Apache Lucene open-source search software
Tests: Remove backward compatibility tests older than previous major version (#15712)
Alexander Reelsenaf85bbcf1c07
diff --git a/lucene/backward-codecs/src/generated/checksums/regenerateForUtil84.json b/lucene/backward-codecs/src/generated/checksums/regenerateForUtil84.jsondeleted file mode 100644index 7e3eacb2c29..00000000000--- a/lucene/backward-codecs/src/generated/checksums/regenerateForUtil84.json+++ /dev/null@@ -1,4 +0,0 @@-{- "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/ForUtil.java": "c3bff5677f7d98fbb362018a4c1dbad4d670610f",- "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/gen_ForUtil.py": "008d3e60b303fbb7c0e8c21f18748a6f384fbbc6"-}diff --git a/lucene/backward-codecs/src/java/module-info.java b/lucene/backward-codecs/src/java/module-info.javaindex dac0bb2f59f..60a0b2897eb 100644--- a/lucene/backward-codecs/src/java/module-info.java+++ b/lucene/backward-codecs/src/java/module-info.java@@ -21,15 +21,7 @@ module org.apache.lucene.backward_codecs {exports org.apache.lucene.backward_codecs;exports org.apache.lucene.backward_codecs.compressing;- exports org.apache.lucene.backward_codecs.lucene40.blocktree;- exports org.apache.lucene.backward_codecs.lucene50;- exports org.apache.lucene.backward_codecs.lucene50.compressing;- exports org.apache.lucene.backward_codecs.lucene60;- exports org.apache.lucene.backward_codecs.lucene70;exports org.apache.lucene.backward_codecs.lucene80;- exports org.apache.lucene.backward_codecs.lucene84;- exports org.apache.lucene.backward_codecs.lucene86;- exports org.apache.lucene.backward_codecs.lucene87;exports org.apache.lucene.backward_codecs.lucene90;exports org.apache.lucene.backward_codecs.lucene90.blocktree;exports org.apache.lucene.backward_codecs.lucene91;@@ -48,8 +40,6 @@ module org.apache.lucene.backward_codecs {provides org.apache.lucene.codecs.DocValuesFormat withorg.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat;provides org.apache.lucene.codecs.PostingsFormat with- org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat,- org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat,org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat,org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat,org.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormat,@@ -66,10 +56,6 @@ module org.apache.lucene.backward_codecs {org.apache.lucene.backward_codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat,org.apache.lucene.backward_codecs.lucene102.Lucene102HnswBinaryQuantizedVectorsFormat;provides org.apache.lucene.codecs.Codec with- org.apache.lucene.backward_codecs.lucene80.Lucene80Codec,- org.apache.lucene.backward_codecs.lucene84.Lucene84Codec,- org.apache.lucene.backward_codecs.lucene86.Lucene86Codec,- org.apache.lucene.backward_codecs.lucene87.Lucene87Codec,org.apache.lucene.backward_codecs.lucene90.Lucene90Codec,org.apache.lucene.backward_codecs.lucene91.Lucene91Codec,org.apache.lucene.backward_codecs.lucene92.Lucene92Codec,diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/CompressionAlgorithm.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/CompressionAlgorithm.javadeleted file mode 100644index 469e773e233..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/CompressionAlgorithm.java+++ /dev/null@@ -1,74 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene40.blocktree;--import java.io.IOException;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.store.DataInput;-import org.apache.lucene.util.compress.LowercaseAsciiCompression;--/** Compression algorithm used for suffixes of a block of terms. */-enum CompressionAlgorithm {- NO_COMPRESSION(0x00) {-- @Override- void read(DataInput in, byte[] out, int len) throws IOException {- in.readBytes(out, 0, len);- }- },-- LOWERCASE_ASCII(0x01) {-- @Override- void read(DataInput in, byte[] out, int len) throws IOException {- LowercaseAsciiCompression.decompress(in, out, len);- }- },-- LZ4(0x02) {-- @Override- void read(DataInput in, byte[] out, int len) throws IOException {- org.apache.lucene.util.compress.LZ4.decompress(- EndiannessReverserUtil.wrapDataInput(in), len, out, 0);- }- };-- private static final CompressionAlgorithm[] BY_CODE = new CompressionAlgorithm[3];-- static {- for (CompressionAlgorithm alg : CompressionAlgorithm.values()) {- BY_CODE[alg.code] = alg;- }- }-- /** Look up a {@link CompressionAlgorithm} by its {@link CompressionAlgorithm#code}. */- static final CompressionAlgorithm byCode(int code) {- if (code < 0 || code >= BY_CODE.length) {- throw new IllegalArgumentException("Illegal code for a compression algorithm: " + code);- }- return BY_CODE[code];- }-- public final int code;-- private CompressionAlgorithm(int code) {- this.code = code;- }-- abstract void read(DataInput in, byte[] out, int len) throws IOException;-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/FieldReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/FieldReader.javadeleted file mode 100644index b74f8b1ecdd..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/FieldReader.java+++ /dev/null@@ -1,219 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene40.blocktree;--import static org.apache.lucene.util.fst.FST.readMetadata;--import java.io.IOException;-import org.apache.lucene.index.FieldInfo;-import org.apache.lucene.index.IndexOptions;-import org.apache.lucene.index.Terms;-import org.apache.lucene.index.TermsEnum;-import org.apache.lucene.store.ByteArrayDataInput;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.automaton.CompiledAutomaton;-import org.apache.lucene.util.fst.ByteSequenceOutputs;-import org.apache.lucene.util.fst.FST;-import org.apache.lucene.util.fst.OffHeapFSTStore;--/**- * BlockTree's implementation of {@link Terms}.- *- * @lucene.internal- */-public final class FieldReader extends Terms {-- // private final boolean DEBUG = BlockTreeTermsWriter.DEBUG;-- final long numTerms;- final FieldInfo fieldInfo;- final long sumTotalTermFreq;- final long sumDocFreq;- final int docCount;- final long rootBlockFP;- final BytesRef rootCode;- final BytesRef minTerm;- final BytesRef maxTerm;- final Lucene40BlockTreeTermsReader parent;-- final FST<BytesRef> index;-- // private boolean DEBUG;-- FieldReader(- Lucene40BlockTreeTermsReader parent,- FieldInfo fieldInfo,- long numTerms,- BytesRef rootCode,- long sumTotalTermFreq,- long sumDocFreq,- int docCount,- long indexStartFP,- IndexInput metaIn,- IndexInput indexIn,- BytesRef minTerm,- BytesRef maxTerm)- throws IOException {- assert numTerms > 0;- this.fieldInfo = fieldInfo;- // DEBUG = BlockTreeTermsReader.DEBUG && fieldInfo.name.equals("id");- this.parent = parent;- this.numTerms = numTerms;- this.sumTotalTermFreq = sumTotalTermFreq;- this.sumDocFreq = sumDocFreq;- this.docCount = docCount;- this.rootCode = rootCode;- this.minTerm = minTerm;- this.maxTerm = maxTerm;- // if (DEBUG) {- // System.out.println("BTTR: seg=" + segment + " field=" + fieldInfo.name + " rootBlockCode="- // + rootCode + " divisor=" + indexDivisor);- // }- rootBlockFP =- (new ByteArrayDataInput(rootCode.bytes, rootCode.offset, rootCode.length)).readVLong()- >>> Lucene40BlockTreeTermsReader.OUTPUT_FLAGS_NUM_BITS;- // Initialize FST always off-heap.- final FST.FSTMetadata<BytesRef> fstMetadata;- if (metaIn == indexIn) { // Only true before Lucene 8.6- final IndexInput clone = indexIn.clone();- clone.seek(indexStartFP);- fstMetadata = readMetadata(clone, ByteSequenceOutputs.getSingleton());- // FST bytes actually only start after the metadata.- indexStartFP = clone.getFilePointer();- } else {- fstMetadata = readMetadata(metaIn, ByteSequenceOutputs.getSingleton());- }- index = FST.fromFSTReader(fstMetadata, new OffHeapFSTStore(indexIn, indexStartFP, fstMetadata));- /*- * if (false) {- * final String dotFileName = segment + "_" + fieldInfo.name + ".dot";- * Writer w = new OutputStreamWriter(new FileOutputStream(dotFileName));- * Util.toDot(index, w, false, false);- * System.out.println("FST INDEX: SAVED to " + dotFileName);- * w.close();- * }- */- }-- @Override- public BytesRef getMin() throws IOException {- if (minTerm == null) {- // Older index that didn't store min/maxTerm- return super.getMin();- } else {- return minTerm;- }- }-- @Override- public BytesRef getMax() throws IOException {- if (maxTerm == null) {- // Older index that didn't store min/maxTerm- return super.getMax();- } else {- return maxTerm;- }- }-- /** For debugging -- used by CheckIndex too */- @Override- public Stats getStats() throws IOException {- return new SegmentTermsEnum(this).computeBlockStats();- }-- @Override- public boolean hasFreqs() {- return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;- }-- @Override- public boolean hasOffsets() {- return fieldInfo- .getIndexOptions()- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)- >= 0;- }-- @Override- public boolean hasPositions() {- return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;- }-- @Override- public boolean hasPayloads() {- return fieldInfo.hasPayloads();- }-- @Override- public TermsEnum iterator() throws IOException {- return new SegmentTermsEnum(this);- }-- @Override- public long size() {- return numTerms;- }-- @Override- public long getSumTotalTermFreq() {- return sumTotalTermFreq;- }-- @Override- public long getSumDocFreq() {- return sumDocFreq;- }-- @Override- public int getDocCount() {- return docCount;- }-- @Override- public TermsEnum intersect(CompiledAutomaton compiled, BytesRef startTerm) throws IOException {- // if (DEBUG) System.out.println(" FieldReader.intersect startTerm=" +- // ToStringUtils.bytesRefToString(startTerm));- // System.out.println("intersect: " + compiled.type + " a=" + compiled.automaton);- // TODO: we could push "it's a range" or "it's a prefix" down into IntersectTermsEnum?- // can we optimize knowing that...?- if (compiled.type != CompiledAutomaton.AUTOMATON_TYPE.NORMAL) {- throw new IllegalArgumentException("please use CompiledAutomaton.getTermsEnum instead");- }- return new IntersectTermsEnum(- this,- compiled.getTransitionAccessor(),- compiled.getByteRunnable(),- compiled.commonSuffixRef,- startTerm);- }-- @Override- public String toString() {- return "BlockTreeTerms(seg="- + parent.segment- + " terms="- + numTerms- + ",postings="- + sumDocFreq- + ",positions="- + sumTotalTermFreq- + ",docs="- + docCount- + ")";- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/IntersectTermsEnum.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/IntersectTermsEnum.javadeleted file mode 100644index 80f47b1eeef..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/IntersectTermsEnum.java+++ /dev/null@@ -1,577 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene40.blocktree;--import java.io.IOException;-import org.apache.lucene.index.BaseTermsEnum;-import org.apache.lucene.index.ImpactsEnum;-import org.apache.lucene.index.PostingsEnum;-import org.apache.lucene.index.TermState;-import org.apache.lucene.index.Terms;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.RamUsageEstimator;-import org.apache.lucene.util.StringHelper;-import org.apache.lucene.util.automaton.ByteRunnable;-import org.apache.lucene.util.automaton.Transition;-import org.apache.lucene.util.automaton.TransitionAccessor;-import org.apache.lucene.util.fst.ByteSequenceOutputs;-import org.apache.lucene.util.fst.FST;-import org.apache.lucene.util.fst.Outputs;--/**- * This is used to implement efficient {@link Terms#intersect} for block-tree. Note that it cannot- * seek, except for the initial term on init. It just "nexts" through the intersection of the- * automaton and the terms. It does not use the terms index at all: on init, it loads the root- * block, and scans its way to the initial term. Likewise, in next it scans until it finds a term- * that matches the current automaton transition.- */-final class IntersectTermsEnum extends BaseTermsEnum {-- // static boolean DEBUG = BlockTreeTermsWriter.DEBUG;-- final IndexInput in;- static final Outputs<BytesRef> fstOutputs = ByteSequenceOutputs.getSingleton();-- IntersectTermsEnumFrame[] stack;-- @SuppressWarnings({"rawtypes", "unchecked"})- private FST.Arc<BytesRef>[] arcs = new FST.Arc[5];-- final ByteRunnable runAutomaton;- final TransitionAccessor automaton;- final BytesRef commonSuffix;-- private IntersectTermsEnumFrame currentFrame;- private Transition currentTransition;-- private final BytesRef term = new BytesRef();-- private final FST.BytesReader fstReader;-- final FieldReader fr;-- private BytesRef savedStartTerm;-- // TODO: in some cases we can filter by length? eg- // regexp foo*bar must be at least length 6 bytes- public IntersectTermsEnum(- FieldReader fr,- TransitionAccessor automaton,- ByteRunnable runAutomaton,- BytesRef commonSuffix,- BytesRef startTerm)- throws IOException {- this.fr = fr;-- assert automaton != null;- assert runAutomaton != null;-- this.runAutomaton = runAutomaton;- this.automaton = automaton;- this.commonSuffix = commonSuffix;-- in = fr.parent.termsIn.clone();- stack = new IntersectTermsEnumFrame[5];- for (int idx = 0; idx < stack.length; idx++) {- stack[idx] = new IntersectTermsEnumFrame(this, idx);- }- for (int arcIdx = 0; arcIdx < arcs.length; arcIdx++) {- arcs[arcIdx] = new FST.Arc<>();- }-- fstReader = fr.index.getBytesReader();-- // TODO: if the automaton is "smallish" we really- // should use the terms index to seek at least to- // the initial term and likely to subsequent terms- // (or, maybe just fallback to ATE for such cases).- // Else the seek cost of loading the frames will be- // too costly.-- final FST.Arc<BytesRef> arc = fr.index.getFirstArc(arcs[0]);- // Empty string prefix must have an output in the index!- assert arc.isFinal();-- // Special pushFrame since it's the first one:- final IntersectTermsEnumFrame f = stack[0];- f.fp = f.fpOrig = fr.rootBlockFP;- f.prefix = 0;- f.setState(0);- f.arc = arc;- f.outputPrefix = arc.output();- f.load(fr.rootCode);-- // for assert:- assert setSavedStartTerm(startTerm);-- currentFrame = f;- if (startTerm != null) {- seekToStartTerm(startTerm);- }- currentTransition = currentFrame.transition;- }-- // only for assert:- private boolean setSavedStartTerm(BytesRef startTerm) {- savedStartTerm = startTerm == null ? null : BytesRef.deepCopyOf(startTerm);- return true;- }-- @Override- public TermState termState() throws IOException {- currentFrame.decodeMetaData();- return currentFrame.termState.clone();- }-- private IntersectTermsEnumFrame getFrame(int ord) throws IOException {- if (ord >= stack.length) {- final IntersectTermsEnumFrame[] next =- new IntersectTermsEnumFrame- [ArrayUtil.oversize(1 + ord, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];- System.arraycopy(stack, 0, next, 0, stack.length);- for (int stackOrd = stack.length; stackOrd < next.length; stackOrd++) {- next[stackOrd] = new IntersectTermsEnumFrame(this, stackOrd);- }- stack = next;- }- assert stack[ord].ord == ord;- return stack[ord];- }-- private FST.Arc<BytesRef> getArc(int ord) {- if (ord >= arcs.length) {- @SuppressWarnings({"rawtypes", "unchecked"})- final FST.Arc<BytesRef>[] next =- new FST.Arc[ArrayUtil.oversize(1 + ord, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];- System.arraycopy(arcs, 0, next, 0, arcs.length);- for (int arcOrd = arcs.length; arcOrd < next.length; arcOrd++) {- next[arcOrd] = new FST.Arc<>();- }- arcs = next;- }- return arcs[ord];- }-- private IntersectTermsEnumFrame pushFrame(int state) throws IOException {- assert currentFrame != null;-- final IntersectTermsEnumFrame f = getFrame(currentFrame == null ? 0 : 1 + currentFrame.ord);-- f.fp = f.fpOrig = currentFrame.lastSubFP;- f.prefix = currentFrame.prefix + currentFrame.suffix;- f.setState(state);-- // Walk the arc through the index -- we only- // "bother" with this so we can get the floor data- // from the index and skip floor blocks when- // possible:- FST.Arc<BytesRef> arc = currentFrame.arc;- int idx = currentFrame.prefix;- assert currentFrame.suffix > 0;- BytesRef output = currentFrame.outputPrefix;- while (idx < f.prefix) {- final int target = term.bytes[idx] & 0xff;- // TODO: we could be more efficient for the next()- // case by using current arc as starting point,- // passed to findTargetArc- arc = fr.index.findTargetArc(target, arc, getArc(1 + idx), fstReader);- assert arc != null;- output = fstOutputs.add(output, arc.output());- idx++;- }-- f.arc = arc;- f.outputPrefix = output;- assert arc.isFinal();- f.load(fstOutputs.add(output, arc.nextFinalOutput()));- return f;- }-- @Override- public BytesRef term() {- return term;- }-- @Override- public int docFreq() throws IOException {- currentFrame.decodeMetaData();- return currentFrame.termState.docFreq;- }-- @Override- public long totalTermFreq() throws IOException {- currentFrame.decodeMetaData();- return currentFrame.termState.totalTermFreq;- }-- @Override- public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException {- currentFrame.decodeMetaData();- return fr.parent.postingsReader.postings(fr.fieldInfo, currentFrame.termState, reuse, flags);- }-- @Override- public ImpactsEnum impacts(int flags) throws IOException {- currentFrame.decodeMetaData();- return fr.parent.postingsReader.impacts(fr.fieldInfo, currentFrame.termState, flags);- }-- private int getState() {- int state = currentFrame.state;- for (int idx = 0; idx < currentFrame.suffix; idx++) {- state =- runAutomaton.step(- state, currentFrame.suffixBytes[currentFrame.startBytePos + idx] & 0xff);- assert state != -1;- }- return state;- }-- // NOTE: specialized to only doing the first-time- // seek, but we could generalize it to allow- // arbitrary seekExact/Ceil. Note that this is a- // seekFloor!- private void seekToStartTerm(BytesRef target) throws IOException {- assert currentFrame.ord == 0;- if (term.length < target.length) {- term.bytes = ArrayUtil.grow(term.bytes, target.length);- }- FST.Arc<BytesRef> arc = arcs[0];- assert arc == currentFrame.arc;-- for (int idx = 0; idx <= target.length; idx++) {-- while (true) {- final int savNextEnt = currentFrame.nextEnt;- final int savePos = currentFrame.suffixesReader.getPosition();- final int saveLengthPos = currentFrame.suffixLengthsReader.getPosition();- final int saveStartBytePos = currentFrame.startBytePos;- final int saveSuffix = currentFrame.suffix;- final long saveLastSubFP = currentFrame.lastSubFP;- final int saveTermBlockOrd = currentFrame.termState.termBlockOrd;-- final boolean isSubBlock = currentFrame.next();-- term.length = currentFrame.prefix + currentFrame.suffix;- if (term.bytes.length < term.length) {- term.bytes = ArrayUtil.grow(term.bytes, term.length);- }- System.arraycopy(- currentFrame.suffixBytes,- currentFrame.startBytePos,- term.bytes,- currentFrame.prefix,- currentFrame.suffix);-- if (isSubBlock && StringHelper.startsWith(target, term)) {- // Recurse- currentFrame = pushFrame(getState());- break;- } else {- final int cmp = term.compareTo(target);- if (cmp < 0) {- if (currentFrame.nextEnt == currentFrame.entCount) {- if (!currentFrame.isLastInFloor) {- // Advance to next floor block- currentFrame.loadNextFloorBlock();- continue;- } else {- return;- }- }- continue;- } else if (cmp == 0) {- return;- } else {- // Fallback to prior entry: the semantics of- // this method is that the first call to- // next() will return the term after the- // requested term- currentFrame.nextEnt = savNextEnt;- currentFrame.lastSubFP = saveLastSubFP;- currentFrame.startBytePos = saveStartBytePos;- currentFrame.suffix = saveSuffix;- currentFrame.suffixesReader.setPosition(savePos);- currentFrame.suffixLengthsReader.setPosition(saveLengthPos);- currentFrame.termState.termBlockOrd = saveTermBlockOrd;- System.arraycopy(- currentFrame.suffixBytes,- currentFrame.startBytePos,- term.bytes,- currentFrame.prefix,- currentFrame.suffix);- term.length = currentFrame.prefix + currentFrame.suffix;- // If the last entry was a block we don't- // need to bother recursing and pushing to- // the last term under it because the first- // next() will simply skip the frame anyway- return;- }- }- }- }-- assert false;- }-- private boolean popPushNext() throws IOException {- // Pop finished frames- while (currentFrame.nextEnt == currentFrame.entCount) {- if (!currentFrame.isLastInFloor) {- // Advance to next floor block- currentFrame.loadNextFloorBlock();- break;- } else {- if (currentFrame.ord == 0) {- throw NoMoreTermsException.INSTANCE;- }- final long lastFP = currentFrame.fpOrig;- currentFrame = stack[currentFrame.ord - 1];- currentTransition = currentFrame.transition;- assert currentFrame.lastSubFP == lastFP;- }- }-- return currentFrame.next();- }-- // Only used internally when there are no more terms in next():- private static final class NoMoreTermsException extends RuntimeException {-- // Only used internally when there are no more terms in next():- public static final NoMoreTermsException INSTANCE = new NoMoreTermsException();-- private NoMoreTermsException() {}-- @Override- public Throwable fillInStackTrace() {- // Do nothing:- return this;- }- }-- @Override- public BytesRef next() throws IOException {- try {- return _next();- } catch (NoMoreTermsException _) {- // Provoke NPE if we are (illegally!) called again:- currentFrame = null;- return null;- }- }-- private BytesRef _next() throws IOException {-- boolean isSubBlock = popPushNext();-- nextTerm:- while (true) {- assert currentFrame.transition == currentTransition;-- int state;- int lastState;-- // NOTE: suffix == 0 can only happen on the first term in a block, when- // there is a term exactly matching a prefix in the index. If we- // could somehow re-org the code so we only checked this case immediately- // after pushing a frame...- if (currentFrame.suffix != 0) {-- final byte[] suffixBytes = currentFrame.suffixBytes;-- // This is the first byte of the suffix of the term we are now on:- final int label = suffixBytes[currentFrame.startBytePos] & 0xff;-- if (label < currentTransition.min) {- // Common case: we are scanning terms in this block to "catch up" to- // current transition in the automaton:- int minTrans = currentTransition.min;- while (currentFrame.nextEnt < currentFrame.entCount) {- isSubBlock = currentFrame.next();- if ((suffixBytes[currentFrame.startBytePos] & 0xff) >= minTrans) {- continue nextTerm;- }- }-- // End of frame:- isSubBlock = popPushNext();- continue nextTerm;- }-- // Advance where we are in the automaton to match this label:-- while (label > currentTransition.max) {- if (currentFrame.transitionIndex >= currentFrame.transitionCount - 1) {- // Pop this frame: no further matches are possible because- // we've moved beyond what the max transition will allow- if (currentFrame.ord == 0) {- // Provoke NPE if we are (illegally!) called again:- currentFrame = null;- return null;- }- currentFrame = stack[currentFrame.ord - 1];- currentTransition = currentFrame.transition;- isSubBlock = popPushNext();- continue nextTerm;- }- currentFrame.transitionIndex++;- automaton.getNextTransition(currentTransition);-- if (label < currentTransition.min) {- int minTrans = currentTransition.min;- while (currentFrame.nextEnt < currentFrame.entCount) {- isSubBlock = currentFrame.next();- if ((suffixBytes[currentFrame.startBytePos] & 0xff) >= minTrans) {- continue nextTerm;- }- }-- // End of frame:- isSubBlock = popPushNext();- continue nextTerm;- }- }-- if (commonSuffix != null && !isSubBlock) {- final int termLen = currentFrame.prefix + currentFrame.suffix;- if (termLen < commonSuffix.length) {- // No match- isSubBlock = popPushNext();- continue nextTerm;- }-- final byte[] commonSuffixBytes = commonSuffix.bytes;-- final int lenInPrefix = commonSuffix.length - currentFrame.suffix;- assert commonSuffix.offset == 0;- int suffixBytesPos;- int commonSuffixBytesPos = 0;-- if (lenInPrefix > 0) {- // A prefix of the common suffix overlaps with- // the suffix of the block prefix so we first- // test whether the prefix part matches:- final byte[] termBytes = term.bytes;- int termBytesPos = currentFrame.prefix - lenInPrefix;- assert termBytesPos >= 0;- final int termBytesPosEnd = currentFrame.prefix;- while (termBytesPos < termBytesPosEnd) {- if (termBytes[termBytesPos++] != commonSuffixBytes[commonSuffixBytesPos++]) {- isSubBlock = popPushNext();- continue nextTerm;- }- }- suffixBytesPos = currentFrame.startBytePos;- } else {- suffixBytesPos = currentFrame.startBytePos + currentFrame.suffix - commonSuffix.length;- }-- // Test overlapping suffix part:- final int commonSuffixBytesPosEnd = commonSuffix.length;- while (commonSuffixBytesPos < commonSuffixBytesPosEnd) {- if (suffixBytes[suffixBytesPos++] != commonSuffixBytes[commonSuffixBytesPos++]) {- isSubBlock = popPushNext();- continue nextTerm;- }- }- }-- // TODO: maybe we should do the same linear test- // that AutomatonTermsEnum does, so that if we- // reach a part of the automaton where .* is- // "temporarily" accepted, we just blindly .next()- // until the limit-- // See if the term suffix matches the automaton:-- // We know from above that the first byte in our suffix (label) matches- // the current transition, so we step from the 2nd byte- // in the suffix:- lastState = currentFrame.state;- state = currentTransition.dest;-- int end = currentFrame.startBytePos + currentFrame.suffix;- for (int idx = currentFrame.startBytePos + 1; idx < end; idx++) {- lastState = state;- state = runAutomaton.step(state, suffixBytes[idx] & 0xff);- if (state == -1) {- // No match- isSubBlock = popPushNext();- continue nextTerm;- }- }- } else {- state = currentFrame.state;- lastState = currentFrame.lastState;- }-- if (isSubBlock) {- // Match! Recurse:- copyTerm();- currentFrame = pushFrame(state);- currentTransition = currentFrame.transition;- currentFrame.lastState = lastState;- } else if (runAutomaton.isAccept(state)) {- copyTerm();- assert savedStartTerm == null || term.compareTo(savedStartTerm) > 0- : "saveStartTerm=" + savedStartTerm.utf8ToString() + " term=" + term.utf8ToString();- return term;- } else {- // This term is a prefix of a term accepted by the automaton, but is not itself accepted- }-- isSubBlock = popPushNext();- }- }-- private void copyTerm() {- final int len = currentFrame.prefix + currentFrame.suffix;- if (term.bytes.length < len) {- term.bytes = ArrayUtil.grow(term.bytes, len);- }- System.arraycopy(- currentFrame.suffixBytes,- currentFrame.startBytePos,- term.bytes,- currentFrame.prefix,- currentFrame.suffix);- term.length = len;- }-- @Override- public boolean seekExact(BytesRef text) {- throw new UnsupportedOperationException();- }-- @Override- public void seekExact(long ord) {- throw new UnsupportedOperationException();- }-- @Override- public long ord() {- throw new UnsupportedOperationException();- }-- @Override- public SeekStatus seekCeil(BytesRef text) {- throw new UnsupportedOperationException();- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/IntersectTermsEnumFrame.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/IntersectTermsEnumFrame.javadeleted file mode 100644index d4c470c40a0..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/IntersectTermsEnumFrame.java+++ /dev/null@@ -1,357 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene40.blocktree;--import java.io.IOException;-import java.util.Arrays;-import org.apache.lucene.codecs.BlockTermState;-import org.apache.lucene.index.CorruptIndexException;-import org.apache.lucene.index.IndexOptions;-import org.apache.lucene.store.ByteArrayDataInput;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.automaton.Transition;-import org.apache.lucene.util.fst.FST;--// TODO: can we share this with the frame in STE?-final class IntersectTermsEnumFrame {- final int ord;- long fp;- long fpOrig;- long fpEnd;- long lastSubFP;-- // private static boolean DEBUG = IntersectTermsEnum.DEBUG;-- // State in automaton- int state;-- // State just before the last label- int lastState;-- int metaDataUpto;-- byte[] suffixBytes = new byte[128];- final ByteArrayDataInput suffixesReader = new ByteArrayDataInput();-- byte[] suffixLengthBytes;- final ByteArrayDataInput suffixLengthsReader;-- byte[] statBytes = new byte[64];- int statsSingletonRunLength = 0;- final ByteArrayDataInput statsReader = new ByteArrayDataInput();-- byte[] floorData = new byte[32];- final ByteArrayDataInput floorDataReader = new ByteArrayDataInput();-- // Length of prefix shared by all terms in this block- int prefix;-- // Number of entries (term or sub-block) in this block- int entCount;-- // Which term we will next read- int nextEnt;-- // True if this block is either not a floor block,- // or, it's the last sub-block of a floor block- boolean isLastInFloor;-- // True if all entries are terms- boolean isLeafBlock;-- int numFollowFloorBlocks;- int nextFloorLabel;-- final Transition transition = new Transition();- int transitionIndex;- int transitionCount;-- FST.Arc<BytesRef> arc;-- final BlockTermState termState;-- // metadata buffer- byte[] bytes = new byte[32];-- final ByteArrayDataInput bytesReader = new ByteArrayDataInput();-- // Cumulative output so far- BytesRef outputPrefix;-- int startBytePos;- int suffix;-- private final IntersectTermsEnum ite;- private final int version;-- public IntersectTermsEnumFrame(IntersectTermsEnum ite, int ord) throws IOException {- this.ite = ite;- this.ord = ord;- this.termState = ite.fr.parent.postingsReader.newTermState();- this.termState.totalTermFreq = -1;- this.version = ite.fr.parent.version;- if (version >= Lucene40BlockTreeTermsReader.VERSION_COMPRESSED_SUFFIXES) {- suffixLengthBytes = new byte[32];- suffixLengthsReader = new ByteArrayDataInput();- } else {- suffixLengthBytes = null;- suffixLengthsReader = suffixesReader;- }- }-- void loadNextFloorBlock() throws IOException {- assert numFollowFloorBlocks > 0 : "nextFloorLabel=" + nextFloorLabel;-- do {- fp = fpOrig + (floorDataReader.readVLong() >>> 1);- numFollowFloorBlocks--;- if (numFollowFloorBlocks != 0) {- nextFloorLabel = floorDataReader.readByte() & 0xff;- } else {- nextFloorLabel = 256;- }- } while (numFollowFloorBlocks != 0 && nextFloorLabel <= transition.min);-- load(null);- }-- public void setState(int state) {- this.state = state;- transitionIndex = 0;- transitionCount = ite.automaton.getNumTransitions(state);- if (transitionCount != 0) {- ite.automaton.initTransition(state, transition);- ite.automaton.getNextTransition(transition);- } else {-- // Must set min to -1 so the "label < min" check never falsely triggers:- transition.min = -1;-- // Must set max to -1 so we immediately realize we need to step to the next transition and- // then pop this frame:- transition.max = -1;- }- }-- void load(BytesRef frameIndexData) throws IOException {- if (frameIndexData != null) {- floorDataReader.reset(frameIndexData.bytes, frameIndexData.offset, frameIndexData.length);- // Skip first long -- has redundant fp, hasTerms- // flag, isFloor flag- final long code = floorDataReader.readVLong();- if ((code & Lucene40BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR) != 0) {- // Floor frame- numFollowFloorBlocks = floorDataReader.readVInt();- nextFloorLabel = floorDataReader.readByte() & 0xff;-- // If current state is not accept, and has transitions, we must process- // first block in case it has empty suffix:- if (ite.runAutomaton.isAccept(state) == false && transitionCount != 0) {- // Maybe skip floor blocks:- assert transitionIndex == 0 : "transitionIndex=" + transitionIndex;- while (numFollowFloorBlocks != 0 && nextFloorLabel <= transition.min) {- fp = fpOrig + (floorDataReader.readVLong() >>> 1);- numFollowFloorBlocks--;- if (numFollowFloorBlocks != 0) {- nextFloorLabel = floorDataReader.readByte() & 0xff;- } else {- nextFloorLabel = 256;- }- }- }- }- }-- ite.in.seek(fp);- int code = ite.in.readVInt();- entCount = code >>> 1;- assert entCount > 0;- isLastInFloor = (code & 1) != 0;-- // term suffixes:- if (version >= Lucene40BlockTreeTermsReader.VERSION_COMPRESSED_SUFFIXES) {- final long codeL = ite.in.readVLong();- isLeafBlock = (codeL & 0x04) != 0;- final int numSuffixBytes = (int) (codeL >>> 3);- if (suffixBytes.length < numSuffixBytes) {- suffixBytes = new byte[ArrayUtil.oversize(numSuffixBytes, 1)];- }- final CompressionAlgorithm compressionAlg;- try {- compressionAlg = CompressionAlgorithm.byCode((int) codeL & 0x03);- } catch (IllegalArgumentException e) {- throw new CorruptIndexException(e.getMessage(), ite.in, e);- }- compressionAlg.read(ite.in, suffixBytes, numSuffixBytes);- suffixesReader.reset(suffixBytes, 0, numSuffixBytes);-- int numSuffixLengthBytes = ite.in.readVInt();- final boolean allEqual = (numSuffixLengthBytes & 0x01) != 0;- numSuffixLengthBytes >>>= 1;- if (suffixLengthBytes.length < numSuffixLengthBytes) {- suffixLengthBytes = new byte[ArrayUtil.oversize(numSuffixLengthBytes, 1)];- }- if (allEqual) {- Arrays.fill(suffixLengthBytes, 0, numSuffixLengthBytes, ite.in.readByte());- } else {- ite.in.readBytes(suffixLengthBytes, 0, numSuffixLengthBytes);- }- suffixLengthsReader.reset(suffixLengthBytes, 0, numSuffixLengthBytes);- } else {- code = ite.in.readVInt();- isLeafBlock = (code & 1) != 0;- int numBytes = code >>> 1;- if (suffixBytes.length < numBytes) {- suffixBytes = new byte[ArrayUtil.oversize(numBytes, 1)];- }- ite.in.readBytes(suffixBytes, 0, numBytes);- suffixesReader.reset(suffixBytes, 0, numBytes);- }-- // stats- int numBytes = ite.in.readVInt();- if (statBytes.length < numBytes) {- statBytes = new byte[ArrayUtil.oversize(numBytes, 1)];- }- ite.in.readBytes(statBytes, 0, numBytes);- statsReader.reset(statBytes, 0, numBytes);- statsSingletonRunLength = 0;- metaDataUpto = 0;-- termState.termBlockOrd = 0;- nextEnt = 0;-- // metadata- numBytes = ite.in.readVInt();- if (bytes.length < numBytes) {- bytes = new byte[ArrayUtil.oversize(numBytes, 1)];- }- ite.in.readBytes(bytes, 0, numBytes);- bytesReader.reset(bytes, 0, numBytes);-- if (!isLastInFloor) {- // Sub-blocks of a single floor block are always- // written one after another -- tail recurse:- fpEnd = ite.in.getFilePointer();- }- }-- // TODO: maybe add scanToLabel; should give perf boost-- // Decodes next entry; returns true if it's a sub-block- public boolean next() {- if (isLeafBlock) {- nextLeaf();- return false;- } else {- return nextNonLeaf();- }- }-- public void nextLeaf() {- assert nextEnt != -1 && nextEnt < entCount- : "nextEnt=" + nextEnt + " entCount=" + entCount + " fp=" + fp;- nextEnt++;- suffix = suffixLengthsReader.readVInt();- startBytePos = suffixesReader.getPosition();- suffixesReader.skipBytes(suffix);- }-- public boolean nextNonLeaf() {- assert nextEnt != -1 && nextEnt < entCount- : "nextEnt=" + nextEnt + " entCount=" + entCount + " fp=" + fp;- nextEnt++;- final int code = suffixLengthsReader.readVInt();- suffix = code >>> 1;- startBytePos = suffixesReader.getPosition();- suffixesReader.skipBytes(suffix);- if ((code & 1) == 0) {- // A normal term- termState.termBlockOrd++;- return false;- } else {- // A sub-block; make sub-FP absolute:- lastSubFP = fp - suffixLengthsReader.readVLong();- return true;- }- }-- public int getTermBlockOrd() {- return isLeafBlock ? nextEnt : termState.termBlockOrd;- }-- public void decodeMetaData() throws IOException {-- // lazily catch up on metadata decode:- final int limit = getTermBlockOrd();- boolean absolute = metaDataUpto == 0;- assert limit > 0;-- // TODO: better API would be "jump straight to term=N"???- while (metaDataUpto < limit) {-- // TODO: we could make "tiers" of metadata, ie,- // decode docFreq/totalTF but don't decode postings- // metadata; this way caller could get- // docFreq/totalTF w/o paying decode cost for- // postings-- // TODO: if docFreq were bulk decoded we could- // just skipN here:-- // stats- if (version >= Lucene40BlockTreeTermsReader.VERSION_COMPRESSED_SUFFIXES) {- if (statsSingletonRunLength > 0) {- termState.docFreq = 1;- termState.totalTermFreq = 1;- statsSingletonRunLength--;- } else {- int token = statsReader.readVInt();- if (version >= Lucene40BlockTreeTermsReader.VERSION_COMPRESSED_SUFFIXES- && (token & 1) == 1) {- termState.docFreq = 1;- termState.totalTermFreq = 1;- statsSingletonRunLength = token >>> 1;- } else {- termState.docFreq = token >>> 1;- if (ite.fr.fieldInfo.getIndexOptions() == IndexOptions.DOCS) {- termState.totalTermFreq = termState.docFreq;- } else {- termState.totalTermFreq = termState.docFreq + statsReader.readVLong();- }- }- }- } else {- termState.docFreq = statsReader.readVInt();- // if (DEBUG) System.out.println(" dF=" + state.docFreq);- if (ite.fr.fieldInfo.getIndexOptions() == IndexOptions.DOCS) {- termState.totalTermFreq = termState.docFreq; // all postings have freq=1- } else {- termState.totalTermFreq = termState.docFreq + statsReader.readVLong();- // if (DEBUG) System.out.println(" totTF=" + state.totalTermFreq);- }- }- // metadata- ite.fr.parent.postingsReader.decodeTerm(bytesReader, ite.fr.fieldInfo, termState, absolute);-- metaDataUpto++;- absolute = false;- }- termState.termBlockOrd = metaDataUpto;- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/Lucene40BlockTreeTermsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/Lucene40BlockTreeTermsReader.javadeleted file mode 100644index 8d578b8fdca..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/Lucene40BlockTreeTermsReader.java+++ /dev/null@@ -1,378 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene40.blocktree;--import java.io.IOException;-import java.util.ArrayList;-import java.util.Collections;-import java.util.HashMap;-import java.util.Iterator;-import java.util.List;-import java.util.Map;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.FieldsProducer;-import org.apache.lucene.codecs.PostingsReaderBase;-import org.apache.lucene.index.CorruptIndexException;-import org.apache.lucene.index.FieldInfo;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.IndexOptions;-import org.apache.lucene.index.SegmentReadState;-import org.apache.lucene.index.Terms;-import org.apache.lucene.store.ChecksumIndexInput;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.IOUtils;-import org.apache.lucene.util.fst.ByteSequenceOutputs;-import org.apache.lucene.util.fst.Outputs;--/**- * A block-based terms index and dictionary that assigns terms to variable length blocks according- * to how they share prefixes. The terms index is a prefix trie whose leaves are term blocks. The- * advantage of this approach is that seekExact is often able to determine a term cannot exist- * without doing any IO, and intersection with Automata is very fast. Note that this terms- * dictionary has its own fixed terms index (ie, it does not support a pluggable terms index- * implementation).- *- * <p><b>NOTE</b>: this terms dictionary supports min/maxItemsPerBlock during indexing to control- * how much memory the terms index uses.- *- * <p>The data structure used by this implementation is very similar to a burst trie- * (http://citeseer.ist.psu.edu/viewdoc/summary?doi=10.1.1.18.3499), but with added logic to break- * up too-large blocks of all terms sharing a given prefix into smaller ones.- *- * <p>Use {@link org.apache.lucene.index.CheckIndex} with the <code>-verbose</code> option to see- * summary statistics on the blocks in the dictionary.- *- * <p>See {@code BlockTreeTermsWriter}.- *- * @lucene.experimental- */-public final class Lucene40BlockTreeTermsReader extends FieldsProducer {-- static final Outputs<BytesRef> FST_OUTPUTS = ByteSequenceOutputs.getSingleton();-- static final BytesRef NO_OUTPUT = FST_OUTPUTS.getNoOutput();-- static final int OUTPUT_FLAGS_NUM_BITS = 2;- static final int OUTPUT_FLAGS_MASK = 0x3;- static final int OUTPUT_FLAG_IS_FLOOR = 0x1;- static final int OUTPUT_FLAG_HAS_TERMS = 0x2;-- /** Extension of terms file */- static final String TERMS_EXTENSION = "tim";-- static final String TERMS_CODEC_NAME = "BlockTreeTermsDict";-- /** Initial terms format. */- public static final int VERSION_START = 3;-- /** The long[] + byte[] metadata has been replaced with a single byte[]. */- public static final int VERSION_META_LONGS_REMOVED = 4;-- /** Suffixes are compressed to save space. */- public static final int VERSION_COMPRESSED_SUFFIXES = 5;-- /** Metadata is written to its own file. */- public static final int VERSION_META_FILE = 6;-- /** Current terms format. */- public static final int VERSION_CURRENT = VERSION_META_FILE;-- /** Extension of terms index file */- static final String TERMS_INDEX_EXTENSION = "tip";-- static final String TERMS_INDEX_CODEC_NAME = "BlockTreeTermsIndex";-- /** Extension of terms meta file */- static final String TERMS_META_EXTENSION = "tmd";-- static final String TERMS_META_CODEC_NAME = "BlockTreeTermsMeta";-- // Open input to the main terms dict file (_X.tib)- final IndexInput termsIn;- // Open input to the terms index file (_X.tip)- final IndexInput indexIn;-- // private static final boolean DEBUG = BlockTreeTermsWriter.DEBUG;-- // Reads the terms dict entries, to gather state to- // produce DocsEnum on demand- final PostingsReaderBase postingsReader;-- private final Map<String, FieldReader> fieldMap;- private final List<String> fieldList;-- final String segment;-- final int version;-- /** Sole constructor. */- public Lucene40BlockTreeTermsReader(PostingsReaderBase postingsReader, SegmentReadState state)- throws IOException {- boolean success = false;-- this.postingsReader = postingsReader;- this.segment = state.segmentInfo.name;-- try {- String termsName =- IndexFileNames.segmentFileName(segment, state.segmentSuffix, TERMS_EXTENSION);- termsIn = EndiannessReverserUtil.openInput(state.directory, termsName, state.context);- version =- CodecUtil.checkIndexHeader(- termsIn,- TERMS_CODEC_NAME,- VERSION_START,- VERSION_CURRENT,- state.segmentInfo.getId(),- state.segmentSuffix);-- String indexName =- IndexFileNames.segmentFileName(segment, state.segmentSuffix, TERMS_INDEX_EXTENSION);- indexIn = EndiannessReverserUtil.openInput(state.directory, indexName, state.context);- CodecUtil.checkIndexHeader(- indexIn,- TERMS_INDEX_CODEC_NAME,- version,- version,- state.segmentInfo.getId(),- state.segmentSuffix);-- if (version < VERSION_META_FILE) {- // Have PostingsReader init itself- postingsReader.init(termsIn, state);-- // Verifying the checksum against all bytes would be too costly, but for now we at least- // verify proper structure of the checksum footer. This is cheap and can detect some forms- // of corruption such as file truncation.- CodecUtil.retrieveChecksum(indexIn);- CodecUtil.retrieveChecksum(termsIn);- }-- // Read per-field details- String metaName =- IndexFileNames.segmentFileName(segment, state.segmentSuffix, TERMS_META_EXTENSION);- Map<String, FieldReader> fieldMap = null;- Throwable priorE = null;- long indexLength = -1, termsLength = -1;- try (ChecksumIndexInput metaIn =- version >= VERSION_META_FILE- ? EndiannessReverserUtil.openChecksumInput(state.directory, metaName, state.context)- : null) {- try {- final IndexInput indexMetaIn, termsMetaIn;- if (version >= VERSION_META_FILE) {- CodecUtil.checkIndexHeader(- metaIn,- TERMS_META_CODEC_NAME,- version,- version,- state.segmentInfo.getId(),- state.segmentSuffix);- indexMetaIn = termsMetaIn = metaIn;- postingsReader.init(metaIn, state);- } else {- seekDir(termsIn);- seekDir(termsIn);- seekDir(indexIn);- indexMetaIn = indexIn;- termsMetaIn = termsIn;- }-- final int numFields = termsMetaIn.readVInt();- if (numFields < 0) {- throw new CorruptIndexException("invalid numFields: " + numFields, termsMetaIn);- }- fieldMap = HashMap.newHashMap(numFields);- for (int i = 0; i < numFields; ++i) {- final int field = termsMetaIn.readVInt();- final long numTerms = termsMetaIn.readVLong();- if (numTerms <= 0) {- throw new CorruptIndexException(- "Illegal numTerms for field number: " + field, termsMetaIn);- }- final BytesRef rootCode = readBytesRef(termsMetaIn);- final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);- if (fieldInfo == null) {- throw new CorruptIndexException("invalid field number: " + field, termsMetaIn);- }- final long sumTotalTermFreq = termsMetaIn.readVLong();- // when frequencies are omitted, sumDocFreq=sumTotalTermFreq and only one value is- // written.- final long sumDocFreq =- fieldInfo.getIndexOptions() == IndexOptions.DOCS- ? sumTotalTermFreq- : termsMetaIn.readVLong();- final int docCount = termsMetaIn.readVInt();- if (version < VERSION_META_LONGS_REMOVED) {- final int longsSize = termsMetaIn.readVInt();- if (longsSize < 0) {- throw new CorruptIndexException(- "invalid longsSize for field: " + fieldInfo.name + ", longsSize=" + longsSize,- termsMetaIn);- }- }- BytesRef minTerm = readBytesRef(termsMetaIn);- BytesRef maxTerm = readBytesRef(termsMetaIn);- if (docCount < 0- || docCount > state.segmentInfo.maxDoc()) { // #docs with field must be <= #docs- throw new CorruptIndexException(- "invalid docCount: " + docCount + " maxDoc: " + state.segmentInfo.maxDoc(),- termsMetaIn);- }- if (sumDocFreq < docCount) { // #postings must be >= #docs with field- throw new CorruptIndexException(- "invalid sumDocFreq: " + sumDocFreq + " docCount: " + docCount, termsMetaIn);- }- if (sumTotalTermFreq < sumDocFreq) { // #positions must be >= #postings- throw new CorruptIndexException(- "invalid sumTotalTermFreq: " + sumTotalTermFreq + " sumDocFreq: " + sumDocFreq,- termsMetaIn);- }- final long indexStartFP = indexMetaIn.readVLong();- FieldReader previous =- fieldMap.put(- fieldInfo.name,- new FieldReader(- this,- fieldInfo,- numTerms,- rootCode,- sumTotalTermFreq,- sumDocFreq,- docCount,- indexStartFP,- indexMetaIn,- indexIn,- minTerm,- maxTerm));- if (previous != null) {- throw new CorruptIndexException("duplicate field: " + fieldInfo.name, termsMetaIn);- }- }- if (version >= VERSION_META_FILE) {- indexLength = metaIn.readLong();- termsLength = metaIn.readLong();- }- } catch (Throwable exception) {- priorE = exception;- } finally {- if (metaIn != null) {- CodecUtil.checkFooter(metaIn, priorE);- } else if (priorE != null) {- IOUtils.rethrowAlways(priorE);- }- }- }- if (version >= VERSION_META_FILE) {- // At this point the checksum of the meta file has been verified so the lengths are likely- // correct- CodecUtil.retrieveChecksum(indexIn, indexLength);- CodecUtil.retrieveChecksum(termsIn, termsLength);- } else {- assert indexLength == -1 : indexLength;- assert termsLength == -1 : termsLength;- }- List<String> fieldList = new ArrayList<>(fieldMap.keySet());- fieldList.sort(null);- this.fieldMap = fieldMap;- this.fieldList = Collections.unmodifiableList(fieldList);- success = true;- } finally {- if (!success) {- // this.close() will close in:- IOUtils.closeWhileHandlingException(this);- }- }- }-- private static BytesRef readBytesRef(IndexInput in) throws IOException {- int numBytes = in.readVInt();- if (numBytes < 0) {- throw new CorruptIndexException("invalid bytes length: " + numBytes, in);- }-- BytesRef bytes = new BytesRef();- bytes.length = numBytes;- bytes.bytes = new byte[numBytes];- in.readBytes(bytes.bytes, 0, numBytes);-- return bytes;- }-- /** Seek {@code input} to the directory offset. */- private static void seekDir(IndexInput input) throws IOException {- input.seek(input.length() - CodecUtil.footerLength() - 8);- long offset = input.readLong();- input.seek(offset);- }-- // for debugging- // private static String toHex(int v) {- // return "0x" + Integer.toHexString(v);- // }-- @Override- public void close() throws IOException {- try {- IOUtils.close(indexIn, termsIn, postingsReader);- } finally {- // Clear so refs to terms index is GCable even if- // app hangs onto us:- fieldMap.clear();- }- }-- @Override- public Iterator<String> iterator() {- return fieldList.iterator();- }-- @Override- public Terms terms(String field) throws IOException {- assert field != null;- return fieldMap.get(field);- }-- @Override- public int size() {- return fieldMap.size();- }-- @Override- public void checkIntegrity() throws IOException {- // terms index- CodecUtil.checksumEntireFile(indexIn);-- // term dictionary- CodecUtil.checksumEntireFile(termsIn);-- // postings- postingsReader.checkIntegrity();- }-- @Override- public String toString() {- return getClass().getSimpleName()- + "(fields="- + fieldMap.size()- + ",delegate="- + postingsReader- + ")";- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/SegmentTermsEnum.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/SegmentTermsEnum.javadeleted file mode 100644index 224906fa5a0..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/SegmentTermsEnum.java+++ /dev/null@@ -1,1177 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene40.blocktree;--import java.io.IOException;-import java.io.PrintStream;-import org.apache.lucene.codecs.BlockTermState;-import org.apache.lucene.index.BaseTermsEnum;-import org.apache.lucene.index.ImpactsEnum;-import org.apache.lucene.index.PostingsEnum;-import org.apache.lucene.index.TermState;-import org.apache.lucene.store.ByteArrayDataInput;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.BytesRefBuilder;-import org.apache.lucene.util.RamUsageEstimator;-import org.apache.lucene.util.fst.FST;-import org.apache.lucene.util.fst.Util;--/** Iterates through terms in this field. */-final class SegmentTermsEnum extends BaseTermsEnum {-- // Lazy init:- IndexInput in;-- private SegmentTermsEnumFrame[] stack;- private final SegmentTermsEnumFrame staticFrame;- SegmentTermsEnumFrame currentFrame;- boolean termExists;- final FieldReader fr;-- private int targetBeforeCurrentLength;-- // static boolean DEBUG = BlockTreeTermsWriter.DEBUG;-- private final ByteArrayDataInput scratchReader = new ByteArrayDataInput();-- // What prefix of the current term was present in the index; when we only next() through the- // index, this stays at 0. It's only set when- // we seekCeil/Exact:- private int validIndexPrefix;-- // assert only:- private boolean eof;-- final BytesRefBuilder term = new BytesRefBuilder();- private final FST.BytesReader fstReader;-- @SuppressWarnings({"rawtypes", "unchecked"})- private FST.Arc<BytesRef>[] arcs = new FST.Arc[1];-- public SegmentTermsEnum(FieldReader fr) throws IOException {- this.fr = fr;-- // if (DEBUG) {- // System.out.println("BTTR.init seg=" + fr.parent.segment);- // }- stack = new SegmentTermsEnumFrame[0];-- // Used to hold seek by TermState, or cached seek- staticFrame = new SegmentTermsEnumFrame(this, -1);-- if (fr.index == null) {- fstReader = null;- } else {- fstReader = fr.index.getBytesReader();- }-- // Init w/ root block; don't use index since it may- // not (and need not) have been loaded- for (int arcIdx = 0; arcIdx < arcs.length; arcIdx++) {- arcs[arcIdx] = new FST.Arc<>();- }-- currentFrame = staticFrame;- final FST.Arc<BytesRef> arc;- if (fr.index != null) {- arc = fr.index.getFirstArc(arcs[0]);- // Empty string prefix must have an output in the index!- assert arc.isFinal();- } else {- arc = null;- }- // currentFrame = pushFrame(arc, rootCode, 0);- // currentFrame.loadBlock();- validIndexPrefix = 0;- // if (DEBUG) {- // System.out.println("init frame state " + currentFrame.ord);- // printSeekState();- // }-- // System.out.println();- // computeBlockStats().print(System.out);- }-- // Not private to avoid synthetic access$NNN methods- void initIndexInput() {- if (this.in == null) {- this.in = fr.parent.termsIn.clone();- }- }-- /** Runs next() through the entire terms dict, computing aggregate statistics. */- public Stats computeBlockStats() throws IOException {-- Stats stats = new Stats(fr.parent.segment, fr.fieldInfo.name);- if (fr.index != null) {- stats.indexNumBytes = fr.index.ramBytesUsed();- }-- currentFrame = staticFrame;- FST.Arc<BytesRef> arc;- if (fr.index != null) {- arc = fr.index.getFirstArc(arcs[0]);- // Empty string prefix must have an output in the index!- assert arc.isFinal();- } else {- arc = null;- }-- // Empty string prefix must have an output in the- // index!- currentFrame = pushFrame(arc, fr.rootCode, 0);- currentFrame.fpOrig = currentFrame.fp;- currentFrame.loadBlock();- validIndexPrefix = 0;-- stats.startBlock(currentFrame, !currentFrame.isLastInFloor);-- allTerms:- while (true) {-- // Pop finished blocks- while (currentFrame.nextEnt == currentFrame.entCount) {- stats.endBlock(currentFrame);- if (!currentFrame.isLastInFloor) {- // Advance to next floor block- currentFrame.loadNextFloorBlock();- stats.startBlock(currentFrame, true);- break;- } else {- if (currentFrame.ord == 0) {- break allTerms;- }- final long lastFP = currentFrame.fpOrig;- currentFrame = stack[currentFrame.ord - 1];- assert lastFP == currentFrame.lastSubFP;- // if (DEBUG) {- // System.out.println(" reset validIndexPrefix=" + validIndexPrefix);- // }- }- }-- while (true) {- if (currentFrame.next()) {- // Push to new block:- currentFrame = pushFrame(null, currentFrame.lastSubFP, term.length());- currentFrame.fpOrig = currentFrame.fp;- // This is a "next" frame -- even if it's- // floor'd we must pretend it isn't so we don't- // try to scan to the right floor frame:- currentFrame.loadBlock();- stats.startBlock(currentFrame, !currentFrame.isLastInFloor);- } else {- stats.term(term.get());- break;- }- }- }-- stats.finish();-- // Put root frame back:- currentFrame = staticFrame;- if (fr.index != null) {- arc = fr.index.getFirstArc(arcs[0]);- // Empty string prefix must have an output in the index!- assert arc.isFinal();- } else {- arc = null;- }- currentFrame = pushFrame(arc, fr.rootCode, 0);- currentFrame.rewind();- currentFrame.loadBlock();- validIndexPrefix = 0;- term.clear();-- return stats;- }-- private SegmentTermsEnumFrame getFrame(int ord) throws IOException {- if (ord >= stack.length) {- final SegmentTermsEnumFrame[] next =- new SegmentTermsEnumFrame- [ArrayUtil.oversize(1 + ord, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];- System.arraycopy(stack, 0, next, 0, stack.length);- for (int stackOrd = stack.length; stackOrd < next.length; stackOrd++) {- next[stackOrd] = new SegmentTermsEnumFrame(this, stackOrd);- }- stack = next;- }- assert stack[ord].ord == ord;- return stack[ord];- }-- private FST.Arc<BytesRef> getArc(int ord) {- if (ord >= arcs.length) {- @SuppressWarnings({"rawtypes", "unchecked"})- final FST.Arc<BytesRef>[] next =- new FST.Arc[ArrayUtil.oversize(1 + ord, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];- System.arraycopy(arcs, 0, next, 0, arcs.length);- for (int arcOrd = arcs.length; arcOrd < next.length; arcOrd++) {- next[arcOrd] = new FST.Arc<>();- }- arcs = next;- }- return arcs[ord];- }-- // Pushes a frame we seek'd to- SegmentTermsEnumFrame pushFrame(FST.Arc<BytesRef> arc, BytesRef frameData, int length)- throws IOException {- scratchReader.reset(frameData.bytes, frameData.offset, frameData.length);- final long code = scratchReader.readVLong();- final long fpSeek = code >>> Lucene40BlockTreeTermsReader.OUTPUT_FLAGS_NUM_BITS;- final SegmentTermsEnumFrame f = getFrame(1 + currentFrame.ord);- f.hasTerms = (code & Lucene40BlockTreeTermsReader.OUTPUT_FLAG_HAS_TERMS) != 0;- f.hasTermsOrig = f.hasTerms;- f.isFloor = (code & Lucene40BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR) != 0;- if (f.isFloor) {- f.setFloorData(scratchReader, frameData);- }- pushFrame(arc, fpSeek, length);-- return f;- }-- // Pushes next'd frame or seek'd frame; we later- // lazy-load the frame only when needed- SegmentTermsEnumFrame pushFrame(FST.Arc<BytesRef> arc, long fp, int length) throws IOException {- final SegmentTermsEnumFrame f = getFrame(1 + currentFrame.ord);- f.arc = arc;- if (f.fpOrig == fp && f.nextEnt != -1) {- // if (DEBUG) System.out.println(" push reused frame ord=" + f.ord + " fp=" + f.fp +- // " isFloor?=" + f.isFloor + " hasTerms=" + f.hasTerms + " pref=" + term + " nextEnt=" +- // f.nextEnt + " targetBeforeCurrentLength=" + targetBeforeCurrentLength + " term.length=" +- // term.length + " vs prefix=" + f.prefix);- // if (f.prefix > targetBeforeCurrentLength) {- if (f.ord > targetBeforeCurrentLength) {- f.rewind();- } else {- // if (DEBUG) {- // System.out.println(" skip rewind!");- // }- }- assert length == f.prefix;- } else {- f.nextEnt = -1;- f.prefix = length;- f.state.termBlockOrd = 0;- f.fpOrig = f.fp = fp;- f.lastSubFP = -1;- // if (DEBUG) {- // final int sav = term.length;- // term.length = length;- // System.out.println(" push new frame ord=" + f.ord + " fp=" + f.fp + " hasTerms=" +- // f.hasTerms + " isFloor=" + f.isFloor + " pref=" + ToStringUtils.bytesRefToString(term));- // term.length = sav;- // }- }-- return f;- }-- // asserts only- private boolean clearEOF() {- eof = false;- return true;- }-- // asserts only- private boolean setEOF() {- eof = true;- return true;- }-- @Override- public boolean seekExact(BytesRef target) throws IOException {-- if (fr.index == null) {- throw new IllegalStateException("terms index was not loaded");- }-- if (fr.size() > 0 && (target.compareTo(fr.getMin()) < 0 || target.compareTo(fr.getMax()) > 0)) {- return false;- }-- term.grow(1 + target.length);-- assert clearEOF();-- // if (DEBUG) {- // System.out.println("\nBTTR.seekExact seg=" + fr.parent.segment + " target=" +- // fr.fieldInfo.name + ":" + ToStringUtils.bytesRefToString(target) + " current=" +- // ToStringUtils.bytesRefToString(term) + " (exists?=" + termExists +- // ") validIndexPrefix=" + validIndexPrefix);- // printSeekState(System.out);- // }-- FST.Arc<BytesRef> arc;- int targetUpto;- BytesRef output;-- targetBeforeCurrentLength = currentFrame.ord;-- if (currentFrame != staticFrame) {-- // We are already seek'd; find the common- // prefix of new seek term vs current term and- // re-use the corresponding seek state. For- // example, if app first seeks to foobar, then- // seeks to foobaz, we can re-use the seek state- // for the first 5 bytes.-- // if (DEBUG) {- // System.out.println(" re-use current seek state validIndexPrefix=" + validIndexPrefix);- // }-- arc = arcs[0];- assert arc.isFinal();- output = arc.output();- targetUpto = 0;-- SegmentTermsEnumFrame lastFrame = stack[0];- assert validIndexPrefix <= term.length();-- final int targetLimit = Math.min(target.length, validIndexPrefix);-- int cmp = 0;-- // TODO: reverse vLong byte order for better FST- // prefix output sharing-- // First compare up to valid seek frames:- while (targetUpto < targetLimit) {- cmp = (term.byteAt(targetUpto) & 0xFF) - (target.bytes[target.offset + targetUpto] & 0xFF);- // if (DEBUG) {- // System.out.println(" cycle targetUpto=" + targetUpto + " (vs limit=" + targetLimit- // + ") cmp=" + cmp + " (targetLabel=" + (char) (target.bytes[target.offset + targetUpto]) +- // " vs termLabel=" + (char) (term.bytes[targetUpto]) + ")" + " arc.output=" + arc.output- // + " output=" + output);- // }- if (cmp != 0) {- break;- }- arc = arcs[1 + targetUpto];- assert arc.label() == (target.bytes[target.offset + targetUpto] & 0xFF)- : "arc.label="- + (char) arc.label()- + " targetLabel="- + (char) (target.bytes[target.offset + targetUpto] & 0xFF);- if (arc.output() != Lucene40BlockTreeTermsReader.NO_OUTPUT) {- output = Lucene40BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.output());- }- if (arc.isFinal()) {- lastFrame = stack[1 + lastFrame.ord];- }- targetUpto++;- }-- if (cmp == 0) {- final int targetUptoMid = targetUpto;-- // Second compare the rest of the term, but- // don't save arc/output/frame; we only do this- // to find out if the target term is before,- // equal or after the current term- final int targetLimit2 = Math.min(target.length, term.length());- while (targetUpto < targetLimit2) {- cmp =- (term.byteAt(targetUpto) & 0xFF) - (target.bytes[target.offset + targetUpto] & 0xFF);- // if (DEBUG) {- // System.out.println(" cycle2 targetUpto=" + targetUpto + " (vs limit=" +- // targetLimit + ") cmp=" + cmp + " (targetLabel=" + (char) (target.bytes[target.offset +- // targetUpto]) + " vs termLabel=" + (char) (term.bytes[targetUpto]) + ")");- // }- if (cmp != 0) {- break;- }- targetUpto++;- }-- if (cmp == 0) {- cmp = term.length() - target.length;- }- targetUpto = targetUptoMid;- }-- if (cmp < 0) {- // Common case: target term is after current- // term, ie, app is seeking multiple terms- // in sorted order- // if (DEBUG) {- // System.out.println(" target is after current (shares prefixLen=" + targetUpto + ");- // frame.ord=" + lastFrame.ord);- // }- currentFrame = lastFrame;-- } else if (cmp > 0) {- // Uncommon case: target term- // is before current term; this means we can- // keep the currentFrame but we must rewind it- // (so we scan from the start)- targetBeforeCurrentLength = lastFrame.ord;- // if (DEBUG) {- // System.out.println(" target is before current (shares prefixLen=" + targetUpto + ");- // rewind frame ord=" + lastFrame.ord);- // }- currentFrame = lastFrame;- currentFrame.rewind();- } else {- // Target is exactly the same as current term- assert term.length() == target.length;- if (termExists) {- // if (DEBUG) {- // System.out.println(" target is same as current; return true");- // }- return true;- } else {- // if (DEBUG) {- // System.out.println(" target is same as current but term doesn't exist");- // }- }- // validIndexPrefix = currentFrame.depth;- // term.length = target.length;- // return termExists;- }-- } else {-- targetBeforeCurrentLength = -1;- arc = fr.index.getFirstArc(arcs[0]);-- // Empty string prefix must have an output (block) in the index!- assert arc.isFinal();- assert arc.output() != null;-- // if (DEBUG) {- // System.out.println(" no seek state; push root frame");- // }-- output = arc.output();-- currentFrame = staticFrame;-- // term.length = 0;- targetUpto = 0;- currentFrame =- pushFrame(- arc, Lucene40BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.nextFinalOutput()), 0);- }-- // if (DEBUG) {- // System.out.println(" start index loop targetUpto=" + targetUpto + " output=" + output +- // " currentFrame.ord=" + currentFrame.ord + " targetBeforeCurrentLength=" +- // targetBeforeCurrentLength);- // }-- // We are done sharing the common prefix with the incoming target and where we are currently- // seek'd; now continue walking the index:- while (targetUpto < target.length) {-- final int targetLabel = target.bytes[target.offset + targetUpto] & 0xFF;-- final FST.Arc<BytesRef> nextArc =- fr.index.findTargetArc(targetLabel, arc, getArc(1 + targetUpto), fstReader);-- if (nextArc == null) {-- // Index is exhausted- // if (DEBUG) {- // System.out.println(" index: index exhausted label=" + ((char) targetLabel) + " " +- // toHex(targetLabel));- // }-- validIndexPrefix = currentFrame.prefix;- // validIndexPrefix = targetUpto;-- currentFrame.scanToFloorFrame(target);-- if (!currentFrame.hasTerms) {- termExists = false;- term.setByteAt(targetUpto, (byte) targetLabel);- term.setLength(1 + targetUpto);- // if (DEBUG) {- // System.out.println(" FAST NOT_FOUND term=" + ToStringUtils.bytesRefToString(term));- // }- return false;- }-- currentFrame.loadBlock();-- final SeekStatus result = currentFrame.scanToTerm(target, true);- if (result == SeekStatus.FOUND) {- // if (DEBUG) {- // System.out.println(" return FOUND term=" + term.utf8ToString() + " " + term);- // }- return true;- } else {- // if (DEBUG) {- // System.out.println(" got " + result + "; return NOT_FOUND term=" +- // ToStringUtils.bytesRefToString(term));- // }- return false;- }- } else {- // Follow this arc- arc = nextArc;- term.setByteAt(targetUpto, (byte) targetLabel);- // Aggregate output as we go:- assert arc.output() != null;- if (arc.output() != Lucene40BlockTreeTermsReader.NO_OUTPUT) {- output = Lucene40BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.output());- }-- // if (DEBUG) {- // System.out.println(" index: follow label=" + toHex(target.bytes[target.offset +- // targetUpto]&0xff) + " arc.output=" + arc.output + " arc.nfo=" + arc.nextFinalOutput);- // }- targetUpto++;-- if (arc.isFinal()) {- // if (DEBUG) System.out.println(" arc is final!");- currentFrame =- pushFrame(- arc,- Lucene40BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.nextFinalOutput()),- targetUpto);- // if (DEBUG) System.out.println(" curFrame.ord=" + currentFrame.ord + " hasTerms=" +- // currentFrame.hasTerms);- }- }- }-- // validIndexPrefix = targetUpto;- validIndexPrefix = currentFrame.prefix;-- currentFrame.scanToFloorFrame(target);-- // Target term is entirely contained in the index:- if (!currentFrame.hasTerms) {- termExists = false;- term.setLength(targetUpto);- // if (DEBUG) {- // System.out.println(" FAST NOT_FOUND term=" + ToStringUtils.bytesRefToString(term));- // }- return false;- }-- currentFrame.loadBlock();-- final SeekStatus result = currentFrame.scanToTerm(target, true);- if (result == SeekStatus.FOUND) {- // if (DEBUG) {- // System.out.println(" return FOUND term=" + term.utf8ToString() + " " + term);- // }- return true;- } else {- // if (DEBUG) {- // System.out.println(" got result " + result + "; return NOT_FOUND term=" +- // term.utf8ToString());- // }-- return false;- }- }-- @Override- public SeekStatus seekCeil(BytesRef target) throws IOException {-- if (fr.index == null) {- throw new IllegalStateException("terms index was not loaded");- }-- term.grow(1 + target.length);-- assert clearEOF();-- // if (DEBUG) {- // System.out.println("\nBTTR.seekCeil seg=" + fr.parent.segment + " target=" +- // fr.fieldInfo.name + ":" + ToStringUtils.bytesRefToString(target) +- // " current=" + ToStringUtils.bytesRefToString(term)- // + " (exists?=" + termExists + ") validIndexPrefix= " + validIndexPrefix);- // printSeekState(System.out);- // }-- FST.Arc<BytesRef> arc;- int targetUpto;- BytesRef output;-- targetBeforeCurrentLength = currentFrame.ord;-- if (currentFrame != staticFrame) {-- // We are already seek'd; find the common- // prefix of new seek term vs current term and- // re-use the corresponding seek state. For- // example, if app first seeks to foobar, then- // seeks to foobaz, we can re-use the seek state- // for the first 5 bytes.-- // if (DEBUG) {- // System.out.println(" re-use current seek state validIndexPrefix=" + validIndexPrefix);- // }-- arc = arcs[0];- assert arc.isFinal();- output = arc.output();- targetUpto = 0;-- SegmentTermsEnumFrame lastFrame = stack[0];- assert validIndexPrefix <= term.length();-- final int targetLimit = Math.min(target.length, validIndexPrefix);-- int cmp = 0;-- // TODO: we should write our vLong backwards (MSB- // first) to get better sharing from the FST-- // First compare up to valid seek frames:- while (targetUpto < targetLimit) {- cmp = (term.byteAt(targetUpto) & 0xFF) - (target.bytes[target.offset + targetUpto] & 0xFF);- // if (DEBUG) {- // System.out.println(" cycle targetUpto=" + targetUpto + " (vs limit=" + targetLimit +- // ") cmp=" + cmp + " (targetLabel=" + (char) (target.bytes[target.offset + targetUpto]) +- // " vs termLabel=" + (char) (term.byteAt(targetUpto)) + ")" + " arc.output=" + arc.output- // + " output=" + output);- // }- if (cmp != 0) {- break;- }- arc = arcs[1 + targetUpto];- assert arc.label() == (target.bytes[target.offset + targetUpto] & 0xFF)- : "arc.label="- + (char) arc.label()- + " targetLabel="- + (char) (target.bytes[target.offset + targetUpto] & 0xFF);- // TODO: we could save the outputs in local- // byte[][] instead of making new objs ever- // seek; but, often the FST doesn't have any- // shared bytes (but this could change if we- // reverse vLong byte order)- if (arc.output() != Lucene40BlockTreeTermsReader.NO_OUTPUT) {- output = Lucene40BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.output());- }- if (arc.isFinal()) {- lastFrame = stack[1 + lastFrame.ord];- }- targetUpto++;- }-- if (cmp == 0) {- final int targetUptoMid = targetUpto;- // Second compare the rest of the term, but- // don't save arc/output/frame:- final int targetLimit2 = Math.min(target.length, term.length());- while (targetUpto < targetLimit2) {- cmp =- (term.byteAt(targetUpto) & 0xFF) - (target.bytes[target.offset + targetUpto] & 0xFF);- // if (DEBUG) {- // System.out.println(" cycle2 targetUpto=" + targetUpto + " (vs limit=" + targetLimit- // + ") cmp=" + cmp + " (targetLabel=" + (char) (target.bytes[target.offset + targetUpto])- // + " vs termLabel=" + (char) (term.byteAt(targetUpto)) + ")");- // }- if (cmp != 0) {- break;- }- targetUpto++;- }-- if (cmp == 0) {- cmp = term.length() - target.length;- }- targetUpto = targetUptoMid;- }-- if (cmp < 0) {- // Common case: target term is after current- // term, ie, app is seeking multiple terms- // in sorted order- // if (DEBUG) {- // System.out.println(" target is after current (shares prefixLen=" + targetUpto + ");- // clear frame.scanned ord=" + lastFrame.ord);- // }- currentFrame = lastFrame;-- } else if (cmp > 0) {- // Uncommon case: target term- // is before current term; this means we can- // keep the currentFrame but we must rewind it- // (so we scan from the start)- targetBeforeCurrentLength = 0;- // if (DEBUG) {- // System.out.println(" target is before current (shares prefixLen=" + targetUpto + ");- // rewind frame ord=" + lastFrame.ord);- // }- currentFrame = lastFrame;- currentFrame.rewind();- } else {- // Target is exactly the same as current term- assert term.length() == target.length;- if (termExists) {- // if (DEBUG) {- // System.out.println(" target is same as current; return FOUND");- // }- return SeekStatus.FOUND;- } else {- // if (DEBUG) {- // System.out.println(" target is same as current but term doesn't exist");- // }- }- }-- } else {-- targetBeforeCurrentLength = -1;- arc = fr.index.getFirstArc(arcs[0]);-- // Empty string prefix must have an output (block) in the index!- assert arc.isFinal();- assert arc.output() != null;-- // if (DEBUG) {- // System.out.println(" no seek state; push root frame");- // }-- output = arc.output();-- currentFrame = staticFrame;-- // term.length = 0;- targetUpto = 0;- currentFrame =- pushFrame(- arc, Lucene40BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.nextFinalOutput()), 0);- }-- // if (DEBUG) {- // System.out.println(" start index loop targetUpto=" + targetUpto + " output=" + output +- // " currentFrame.ord+1=" + currentFrame.ord + " targetBeforeCurrentLength=" +- // targetBeforeCurrentLength);- // }-- // We are done sharing the common prefix with the incoming target and where we are currently- // seek'd; now continue walking the index:- while (targetUpto < target.length) {-- final int targetLabel = target.bytes[target.offset + targetUpto] & 0xFF;-- final FST.Arc<BytesRef> nextArc =- fr.index.findTargetArc(targetLabel, arc, getArc(1 + targetUpto), fstReader);-- if (nextArc == null) {-- // Index is exhausted- // if (DEBUG) {- // System.out.println(" index: index exhausted label=" + ((char) targetLabel) + " " +- // targetLabel);- // }-- validIndexPrefix = currentFrame.prefix;- // validIndexPrefix = targetUpto;-- currentFrame.scanToFloorFrame(target);-- currentFrame.loadBlock();-- // if (DEBUG) System.out.println(" now scanToTerm");- final SeekStatus result = currentFrame.scanToTerm(target, false);- if (result == SeekStatus.END) {- term.copyBytes(target);- termExists = false;-- if (next() != null) {- // if (DEBUG) {- // System.out.println(" return NOT_FOUND term=" +- // ToStringUtils.bytesRefToString(term));- // }- return SeekStatus.NOT_FOUND;- } else {- // if (DEBUG) {- // System.out.println(" return END");- // }- return SeekStatus.END;- }- } else {- // if (DEBUG) {- // System.out.println(" return " + result + " term=" +- // ToStringUtils.bytesRefToString(term));- // }- return result;- }- } else {- // Follow this arc- term.setByteAt(targetUpto, (byte) targetLabel);- arc = nextArc;- // Aggregate output as we go:- assert arc.output() != null;- if (arc.output() != Lucene40BlockTreeTermsReader.NO_OUTPUT) {- output = Lucene40BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.output());- }-- // if (DEBUG) {- // System.out.println(" index: follow label=" + (target.bytes[target.offset +- // targetUpto]&0xff) + " arc.output=" + arc.output + " arc.nfo=" + arc.nextFinalOutput);- // }- targetUpto++;-- if (arc.isFinal()) {- // if (DEBUG) System.out.println(" arc is final!");- currentFrame =- pushFrame(- arc,- Lucene40BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.nextFinalOutput()),- targetUpto);- // if (DEBUG) System.out.println(" curFrame.ord=" + currentFrame.ord + " hasTerms=" +- // currentFrame.hasTerms);- }- }- }-- // validIndexPrefix = targetUpto;- validIndexPrefix = currentFrame.prefix;-- currentFrame.scanToFloorFrame(target);-- currentFrame.loadBlock();-- final SeekStatus result = currentFrame.scanToTerm(target, false);-- if (result == SeekStatus.END) {- term.copyBytes(target);- termExists = false;- if (next() != null) {- // if (DEBUG) {- // System.out.println(" return NOT_FOUND term=" + term.get().utf8ToString() + " " + term);- // }- return SeekStatus.NOT_FOUND;- } else {- // if (DEBUG) {- // System.out.println(" return END");- // }- return SeekStatus.END;- }- } else {- return result;- }- }-- @SuppressWarnings("unused")- private void printSeekState(PrintStream out) throws IOException {- if (currentFrame == staticFrame) {- out.println(" no prior seek");- } else {- out.println(" prior seek state:");- int ord = 0;- boolean isSeekFrame = true;- while (true) {- SegmentTermsEnumFrame f = getFrame(ord);- assert f != null;- final BytesRef prefix = new BytesRef(term.get().bytes, 0, f.prefix);- if (f.nextEnt == -1) {- out.println(- " frame "- + (isSeekFrame ? "(seek)" : "(next)")- + " ord="- + ord- + " fp="- + f.fp- + (f.isFloor ? (" (fpOrig=" + f.fpOrig + ")") : "")- + " prefixLen="- + f.prefix- + " prefix="- + prefix- + (f.nextEnt == -1 ? "" : (" (of " + f.entCount + ")"))- + " hasTerms="- + f.hasTerms- + " isFloor="- + f.isFloor- + " code="- + ((f.fp << Lucene40BlockTreeTermsReader.OUTPUT_FLAGS_NUM_BITS)- + (f.hasTerms ? Lucene40BlockTreeTermsReader.OUTPUT_FLAG_HAS_TERMS : 0)- + (f.isFloor ? Lucene40BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR : 0))- + " isLastInFloor="- + f.isLastInFloor- + " mdUpto="- + f.metaDataUpto- + " tbOrd="- + f.getTermBlockOrd());- } else {- out.println(- " frame "- + (isSeekFrame ? "(seek, loaded)" : "(next, loaded)")- + " ord="- + ord- + " fp="- + f.fp- + (f.isFloor ? (" (fpOrig=" + f.fpOrig + ")") : "")- + " prefixLen="- + f.prefix- + " prefix="- + prefix- + " nextEnt="- + f.nextEnt- + (f.nextEnt == -1 ? "" : (" (of " + f.entCount + ")"))- + " hasTerms="- + f.hasTerms- + " isFloor="- + f.isFloor- + " code="- + ((f.fp << Lucene40BlockTreeTermsReader.OUTPUT_FLAGS_NUM_BITS)- + (f.hasTerms ? Lucene40BlockTreeTermsReader.OUTPUT_FLAG_HAS_TERMS : 0)- + (f.isFloor ? Lucene40BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR : 0))- + " lastSubFP="- + f.lastSubFP- + " isLastInFloor="- + f.isLastInFloor- + " mdUpto="- + f.metaDataUpto- + " tbOrd="- + f.getTermBlockOrd());- }- if (fr.index != null) {- assert !isSeekFrame || f.arc != null : "isSeekFrame=" + isSeekFrame + " f.arc=" + f.arc;- if (f.prefix > 0 && isSeekFrame && f.arc.label() != (term.byteAt(f.prefix - 1) & 0xFF)) {- out.println(- " broken seek state: arc.label="- + (char) f.arc.label()- + " vs term byte="- + (char) (term.byteAt(f.prefix - 1) & 0xFF));- throw new RuntimeException("seek state is broken");- }- BytesRef output = Util.get(fr.index, prefix);- if (output == null) {- out.println(" broken seek state: prefix is not final in index");- throw new RuntimeException("seek state is broken");- } else if (isSeekFrame && !f.isFloor) {- final ByteArrayDataInput reader =- new ByteArrayDataInput(output.bytes, output.offset, output.length);- final long codeOrig = reader.readVLong();- final long code =- (f.fp << Lucene40BlockTreeTermsReader.OUTPUT_FLAGS_NUM_BITS)- | (f.hasTerms ? Lucene40BlockTreeTermsReader.OUTPUT_FLAG_HAS_TERMS : 0)- | (f.isFloor ? Lucene40BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR : 0);- if (codeOrig != code) {- out.println(- " broken seek state: output code="- + codeOrig- + " doesn't match frame code="- + code);- throw new RuntimeException("seek state is broken");- }- }- }- if (f == currentFrame) {- break;- }- if (f.prefix == validIndexPrefix) {- isSeekFrame = false;- }- ord++;- }- }- }-- /* Decodes only the term bytes of the next term. If caller then asks for- metadata, ie docFreq, totalTermFreq or pulls a D/&PEnum, we then (lazily)- decode all metadata up to the current term. */- @Override- public BytesRef next() throws IOException {- if (in == null) {- // Fresh TermsEnum; seek to first term:- final FST.Arc<BytesRef> arc;- if (fr.index != null) {- arc = fr.index.getFirstArc(arcs[0]);- // Empty string prefix must have an output in the index!- assert arc.isFinal();- } else {- arc = null;- }- currentFrame = pushFrame(arc, fr.rootCode, 0);- currentFrame.loadBlock();- }-- targetBeforeCurrentLength = currentFrame.ord;-- assert !eof;- // if (DEBUG) {- // System.out.println("\nBTTR.next seg=" + fr.parent.segment + " term=" +- // ToStringUtils.bytesRefToString(term) + " termExists?=" + termExists + " field=" +- // fr.fieldInfo.name + " termBlockOrd=" + currentFrame.state.termBlockOrd +- // " validIndexPrefix=" + validIndexPrefix);- // printSeekState(System.out);- // }-- if (currentFrame == staticFrame) {- // If seek was previously called and the term was- // cached, or seek(TermState) was called, usually- // caller is just going to pull a D/&PEnum or get- // docFreq, etc. But, if they then call next(),- // this method catches up all internal state so next()- // works properly:- // if (DEBUG) System.out.println(" re-seek to pending term=" + term.utf8ToString() + " " +- // term);- final boolean result = seekExact(term.get());- assert result;- }-- // Pop finished blocks- while (currentFrame.nextEnt == currentFrame.entCount) {- if (!currentFrame.isLastInFloor) {- // Advance to next floor block- currentFrame.loadNextFloorBlock();- break;- } else {- // if (DEBUG) System.out.println(" pop frame");- if (currentFrame.ord == 0) {- // if (DEBUG) System.out.println(" return null");- assert setEOF();- term.clear();- validIndexPrefix = 0;- currentFrame.rewind();- termExists = false;- return null;- }- final long lastFP = currentFrame.fpOrig;- currentFrame = stack[currentFrame.ord - 1];-- if (currentFrame.nextEnt == -1 || currentFrame.lastSubFP != lastFP) {- // We popped into a frame that's not loaded- // yet or not scan'd to the right entry- currentFrame.scanToFloorFrame(term.get());- currentFrame.loadBlock();- currentFrame.scanToSubBlock(lastFP);- }-- // Note that the seek state (last seek) has been- // invalidated beyond this depth- validIndexPrefix = Math.min(validIndexPrefix, currentFrame.prefix);- // if (DEBUG) {- // System.out.println(" reset validIndexPrefix=" + validIndexPrefix);- // }- }- }-- while (true) {- if (currentFrame.next()) {- // Push to new block:- // if (DEBUG) System.out.println(" push frame");- currentFrame = pushFrame(null, currentFrame.lastSubFP, term.length());- // This is a "next" frame -- even if it's- // floor'd we must pretend it isn't so we don't- // try to scan to the right floor frame:- currentFrame.loadBlock();- } else {- // if (DEBUG) System.out.println(" return term=" + ToStringUtils.bytesRefToString(term) +- // " currentFrame.ord=" + currentFrame.ord);- return term.get();- }- }- }-- @Override- public BytesRef term() {- assert !eof;- return term.get();- }-- @Override- public int docFreq() throws IOException {- assert !eof;- // if (DEBUG) System.out.println("BTR.docFreq");- currentFrame.decodeMetaData();- // if (DEBUG) System.out.println(" return " + currentFrame.state.docFreq);- return currentFrame.state.docFreq;- }-- @Override- public long totalTermFreq() throws IOException {- assert !eof;- currentFrame.decodeMetaData();- return currentFrame.state.totalTermFreq;- }-- @Override- public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException {- assert !eof;- // if (DEBUG) {- // System.out.println("BTTR.docs seg=" + segment);- // }- currentFrame.decodeMetaData();- // if (DEBUG) {- // System.out.println(" state=" + currentFrame.state);- // }- return fr.parent.postingsReader.postings(fr.fieldInfo, currentFrame.state, reuse, flags);- }-- @Override- public ImpactsEnum impacts(int flags) throws IOException {- assert !eof;- // if (DEBUG) {- // System.out.println("BTTR.docs seg=" + segment);- // }- currentFrame.decodeMetaData();- // if (DEBUG) {- // System.out.println(" state=" + currentFrame.state);- // }- return fr.parent.postingsReader.impacts(fr.fieldInfo, currentFrame.state, flags);- }-- @Override- public void seekExact(BytesRef target, TermState otherState) {- // if (DEBUG) {- // System.out.println("BTTR.seekExact termState seg=" + segment + " target=" +- // target.utf8ToString() + " " + target + " state=" + otherState);- // }- assert clearEOF();- if (target.compareTo(term.get()) != 0 || !termExists) {- assert otherState != null && otherState instanceof BlockTermState;- currentFrame = staticFrame;- currentFrame.state.copyFrom(otherState);- term.copyBytes(target);- currentFrame.metaDataUpto = currentFrame.getTermBlockOrd();- assert currentFrame.metaDataUpto > 0;- validIndexPrefix = 0;- } else {- // if (DEBUG) {- // System.out.println(" skip seek: already on target state=" + currentFrame.state);- // }- }- }-- @Override- public TermState termState() throws IOException {- assert !eof;- currentFrame.decodeMetaData();- TermState ts = currentFrame.state.clone();- // if (DEBUG) System.out.println("BTTR.termState seg=" + segment + " state=" + ts);- return ts;- }-- @Override- public void seekExact(long ord) {- throw new UnsupportedOperationException();- }-- @Override- public long ord() {- throw new UnsupportedOperationException();- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/SegmentTermsEnumFrame.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/SegmentTermsEnumFrame.javadeleted file mode 100644index 3003151bbe7..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/SegmentTermsEnumFrame.java+++ /dev/null@@ -1,794 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene40.blocktree;--import java.io.IOException;-import java.util.Arrays;-import org.apache.lucene.codecs.BlockTermState;-import org.apache.lucene.index.CorruptIndexException;-import org.apache.lucene.index.IndexOptions;-import org.apache.lucene.index.TermsEnum.SeekStatus;-import org.apache.lucene.store.ByteArrayDataInput;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.fst.FST;--final class SegmentTermsEnumFrame {- // Our index in stack[]:- final int ord;-- boolean hasTerms;- boolean hasTermsOrig;- boolean isFloor;-- FST.Arc<BytesRef> arc;-- // static boolean DEBUG = BlockTreeTermsWriter.DEBUG;-- // File pointer where this block was loaded from- long fp;- long fpOrig;- long fpEnd;- long totalSuffixBytes; // for stats-- byte[] suffixBytes = new byte[128];- final ByteArrayDataInput suffixesReader = new ByteArrayDataInput();-- byte[] suffixLengthBytes;- final ByteArrayDataInput suffixLengthsReader;-- byte[] statBytes = new byte[64];- int statsSingletonRunLength = 0;- final ByteArrayDataInput statsReader = new ByteArrayDataInput();-- byte[] floorData = new byte[32];- final ByteArrayDataInput floorDataReader = new ByteArrayDataInput();-- // Length of prefix shared by all terms in this block- int prefix;-- // Number of entries (term or sub-block) in this block- int entCount;-- // Which term we will next read, or -1 if the block- // isn't loaded yet- int nextEnt;-- // True if this block is either not a floor block,- // or, it's the last sub-block of a floor block- boolean isLastInFloor;-- // True if all entries are terms- boolean isLeafBlock;-- long lastSubFP;-- int nextFloorLabel;- int numFollowFloorBlocks;-- // Next term to decode metaData; we decode metaData- // lazily so that scanning to find the matching term is- // fast and only if you find a match and app wants the- // stats or docs/positions enums, will we decode the- // metaData- int metaDataUpto;-- final BlockTermState state;-- // metadata buffer- byte[] bytes = new byte[32];- final ByteArrayDataInput bytesReader = new ByteArrayDataInput();-- private final SegmentTermsEnum ste;- private final int version;-- public SegmentTermsEnumFrame(SegmentTermsEnum ste, int ord) throws IOException {- this.ste = ste;- this.ord = ord;- this.state = ste.fr.parent.postingsReader.newTermState();- this.state.totalTermFreq = -1;- this.version = ste.fr.parent.version;- if (version >= Lucene40BlockTreeTermsReader.VERSION_COMPRESSED_SUFFIXES) {- suffixLengthBytes = new byte[32];- suffixLengthsReader = new ByteArrayDataInput();- } else {- suffixLengthBytes = null;- suffixLengthsReader = suffixesReader;- }- }-- public void setFloorData(ByteArrayDataInput in, BytesRef source) {- final int numBytes = source.length - (in.getPosition() - source.offset);- if (numBytes > floorData.length) {- floorData = new byte[ArrayUtil.oversize(numBytes, 1)];- }- System.arraycopy(source.bytes, source.offset + in.getPosition(), floorData, 0, numBytes);- floorDataReader.reset(floorData, 0, numBytes);- numFollowFloorBlocks = floorDataReader.readVInt();- nextFloorLabel = floorDataReader.readByte() & 0xff;- // if (DEBUG) {- // System.out.println(" setFloorData fpOrig=" + fpOrig + " bytes=" + new- // BytesRef(source.bytes, source.offset + in.getPosition(), numBytes) + " numFollowFloorBlocks="- // + numFollowFloorBlocks + " nextFloorLabel=" + toHex(nextFloorLabel));- // }- }-- public int getTermBlockOrd() {- return isLeafBlock ? nextEnt : state.termBlockOrd;- }-- void loadNextFloorBlock() throws IOException {- // if (DEBUG) {- // System.out.println(" loadNextFloorBlock fp=" + fp + " fpEnd=" + fpEnd);- // }- assert arc == null || isFloor : "arc=" + arc + " isFloor=" + isFloor;- fp = fpEnd;- nextEnt = -1;- loadBlock();- }-- /* Does initial decode of next block of terms; this- doesn't actually decode the docFreq, totalTermFreq,- postings details (frq/prx offset, etc.) metadata;- it just loads them as byte[] blobs which are then- decoded on-demand if the metadata is ever requested- for any term in this block. This enables terms-only- intensive consumes (eg certain MTQs, respelling) to- not pay the price of decoding metadata they won't- use. */- void loadBlock() throws IOException {-- // Clone the IndexInput lazily, so that consumers- // that just pull a TermsEnum to- // seekExact(TermState) don't pay this cost:- ste.initIndexInput();-- if (nextEnt != -1) {- // Already loaded- return;- }- // System.out.println("blc=" + blockLoadCount);-- ste.in.seek(fp);- int code = ste.in.readVInt();- entCount = code >>> 1;- assert entCount > 0;- isLastInFloor = (code & 1) != 0;-- assert arc == null || (isLastInFloor || isFloor)- : "fp=" + fp + " arc=" + arc + " isFloor=" + isFloor + " isLastInFloor=" + isLastInFloor;-- // TODO: if suffixes were stored in random-access- // array structure, then we could do binary search- // instead of linear scan to find target term; eg- // we could have simple array of offsets-- final long startSuffixFP = ste.in.getFilePointer();- // term suffixes:- if (version >= Lucene40BlockTreeTermsReader.VERSION_COMPRESSED_SUFFIXES) {- final long codeL = ste.in.readVLong();- isLeafBlock = (codeL & 0x04) != 0;- final int numSuffixBytes = (int) (codeL >>> 3);- if (suffixBytes.length < numSuffixBytes) {- suffixBytes = new byte[ArrayUtil.oversize(numSuffixBytes, 1)];- }- try {- compressionAlg = CompressionAlgorithm.byCode((int) codeL & 0x03);- } catch (IllegalArgumentException e) {- throw new CorruptIndexException(e.getMessage(), ste.in, e);- }- compressionAlg.read(ste.in, suffixBytes, numSuffixBytes);- suffixesReader.reset(suffixBytes, 0, numSuffixBytes);-- int numSuffixLengthBytes = ste.in.readVInt();- final boolean allEqual = (numSuffixLengthBytes & 0x01) != 0;- numSuffixLengthBytes >>>= 1;- if (suffixLengthBytes.length < numSuffixLengthBytes) {- suffixLengthBytes = new byte[ArrayUtil.oversize(numSuffixLengthBytes, 1)];- }- if (allEqual) {- Arrays.fill(suffixLengthBytes, 0, numSuffixLengthBytes, ste.in.readByte());- } else {- ste.in.readBytes(suffixLengthBytes, 0, numSuffixLengthBytes);- }- suffixLengthsReader.reset(suffixLengthBytes, 0, numSuffixLengthBytes);- } else {- code = ste.in.readVInt();- isLeafBlock = (code & 1) != 0;- int numBytes = code >>> 1;- if (suffixBytes.length < numBytes) {- suffixBytes = new byte[ArrayUtil.oversize(numBytes, 1)];- }- ste.in.readBytes(suffixBytes, 0, numBytes);- suffixesReader.reset(suffixBytes, 0, numBytes);- }- totalSuffixBytes = ste.in.getFilePointer() - startSuffixFP;-- /*if (DEBUG) {- if (arc == null) {- System.out.println(" loadBlock (next) fp=" + fp + " entCount=" + entCount + " prefixLen=" + prefix + " isLastInFloor=" + isLastInFloor + " leaf?=" + isLeafBlock);- } else {- System.out.println(" loadBlock (seek) fp=" + fp + " entCount=" + entCount + " prefixLen=" + prefix + " hasTerms?=" + hasTerms + " isFloor?=" + isFloor + " isLastInFloor=" + isLastInFloor + " leaf?=" + isLeafBlock);- }- }*/-- // stats- int numBytes = ste.in.readVInt();- if (statBytes.length < numBytes) {- statBytes = new byte[ArrayUtil.oversize(numBytes, 1)];- }- ste.in.readBytes(statBytes, 0, numBytes);- statsReader.reset(statBytes, 0, numBytes);- statsSingletonRunLength = 0;- metaDataUpto = 0;-- state.termBlockOrd = 0;- nextEnt = 0;- lastSubFP = -1;-- // TODO: we could skip this if !hasTerms; but- // that's rare so won't help much- // metadata- numBytes = ste.in.readVInt();- if (bytes.length < numBytes) {- bytes = new byte[ArrayUtil.oversize(numBytes, 1)];- }- ste.in.readBytes(bytes, 0, numBytes);- bytesReader.reset(bytes, 0, numBytes);-- // Sub-blocks of a single floor block are always- // written one after another -- tail recurse:- fpEnd = ste.in.getFilePointer();- // if (DEBUG) {- // System.out.println(" fpEnd=" + fpEnd);- // }- }-- void rewind() {-- // Force reload:- fp = fpOrig;- nextEnt = -1;- hasTerms = hasTermsOrig;- if (isFloor) {- floorDataReader.rewind();- numFollowFloorBlocks = floorDataReader.readVInt();- assert numFollowFloorBlocks > 0;- nextFloorLabel = floorDataReader.readByte() & 0xff;- }-- /*- //System.out.println("rewind");- // Keeps the block loaded, but rewinds its state:- if (nextEnt > 0 || fp != fpOrig) {- if (DEBUG) {- System.out.println(" rewind frame ord=" + ord + " fpOrig=" + fpOrig + " fp=" + fp + " hasTerms?=" + hasTerms + " isFloor?=" + isFloor + " nextEnt=" + nextEnt + " prefixLen=" + prefix);- }- if (fp != fpOrig) {- fp = fpOrig;- nextEnt = -1;- } else {- nextEnt = 0;- }- hasTerms = hasTermsOrig;- if (isFloor) {- floorDataReader.rewind();- numFollowFloorBlocks = floorDataReader.readVInt();- nextFloorLabel = floorDataReader.readByte() & 0xff;- }- assert suffixBytes != null;- suffixesReader.rewind();- assert statBytes != null;- statsReader.rewind();- metaDataUpto = 0;- state.termBlockOrd = 0;- // TODO: skip this if !hasTerms? Then postings- // impl wouldn't have to write useless 0 byte- postingsReader.resetTermsBlock(fieldInfo, state);- lastSubFP = -1;- } else if (DEBUG) {- System.out.println(" skip rewind fp=" + fp + " fpOrig=" + fpOrig + " nextEnt=" + nextEnt + " ord=" + ord);- }- */- }-- // Decodes next entry; returns true if it's a sub-block- public boolean next() throws IOException {- if (isLeafBlock) {- nextLeaf();- return false;- } else {- return nextNonLeaf();- }- }-- public void nextLeaf() {- // if (DEBUG) System.out.println(" frame.next ord=" + ord + " nextEnt=" + nextEnt +- // " entCount=" + entCount);- assert nextEnt != -1 && nextEnt < entCount- : "nextEnt=" + nextEnt + " entCount=" + entCount + " fp=" + fp;- nextEnt++;- suffix = suffixLengthsReader.readVInt();- startBytePos = suffixesReader.getPosition();- ste.term.setLength(prefix + suffix);- ste.term.grow(ste.term.length());- suffixesReader.readBytes(ste.term.bytes(), prefix, suffix);- ste.termExists = true;- }-- public boolean nextNonLeaf() throws IOException {- // if (DEBUG) System.out.println(" stef.next ord=" + ord + " nextEnt=" + nextEnt + " entCount="- // + entCount + " fp=" + suffixesReader.getPosition());- while (true) {- if (nextEnt == entCount) {- assert arc == null || (isFloor && isLastInFloor == false)- : "isFloor=" + isFloor + " isLastInFloor=" + isLastInFloor;- loadNextFloorBlock();- if (isLeafBlock) {- nextLeaf();- return false;- } else {- continue;- }- }-- assert nextEnt != -1 && nextEnt < entCount- : "nextEnt=" + nextEnt + " entCount=" + entCount + " fp=" + fp;- nextEnt++;- final int code = suffixLengthsReader.readVInt();- suffix = code >>> 1;- startBytePos = suffixesReader.getPosition();- ste.term.setLength(prefix + suffix);- ste.term.grow(ste.term.length());- suffixesReader.readBytes(ste.term.bytes(), prefix, suffix);- if ((code & 1) == 0) {- // A normal term- ste.termExists = true;- subCode = 0;- state.termBlockOrd++;- return false;- } else {- // A sub-block; make sub-FP absolute:- ste.termExists = false;- subCode = suffixLengthsReader.readVLong();- lastSubFP = fp - subCode;- // if (DEBUG) {- // System.out.println(" lastSubFP=" + lastSubFP);- // }- return true;- }- }- }-- // TODO: make this array'd so we can do bin search?- // likely not worth it? need to measure how many- // floor blocks we "typically" get- public void scanToFloorFrame(BytesRef target) {-- if (!isFloor || target.length <= prefix) {- // if (DEBUG) {- // System.out.println(" scanToFloorFrame skip: isFloor=" + isFloor + " target.length=" +- // target.length + " vs prefix=" + prefix);- // }- return;- }-- final int targetLabel = target.bytes[target.offset + prefix] & 0xFF;-- // if (DEBUG) {- // System.out.println(" scanToFloorFrame fpOrig=" + fpOrig + " targetLabel=" +- // toHex(targetLabel) + " vs nextFloorLabel=" + toHex(nextFloorLabel) + " numFollowFloorBlocks="- // + numFollowFloorBlocks);- // }-- if (targetLabel < nextFloorLabel) {- // if (DEBUG) {- // System.out.println(" already on correct block");- // }- return;- }-- assert numFollowFloorBlocks != 0;-- long newFP = fpOrig;- while (true) {- final long code = floorDataReader.readVLong();- newFP = fpOrig + (code >>> 1);- hasTerms = (code & 1) != 0;- // if (DEBUG) {- // System.out.println(" label=" + toHex(nextFloorLabel) + " fp=" + newFP +- // " hasTerms?=" + hasTerms + " numFollowFloor=" + numFollowFloorBlocks);- // }-- isLastInFloor = numFollowFloorBlocks == 1;- numFollowFloorBlocks--;-- if (isLastInFloor) {- nextFloorLabel = 256;- // if (DEBUG) {- // System.out.println(" stop! last block nextFloorLabel=" +- // toHex(nextFloorLabel));- // }- break;- } else {- nextFloorLabel = floorDataReader.readByte() & 0xff;- if (targetLabel < nextFloorLabel) {- // if (DEBUG) {- // System.out.println(" stop! nextFloorLabel=" + toHex(nextFloorLabel));- // }- break;- }- }- }-- if (newFP != fp) {- // Force re-load of the block:- // if (DEBUG) {- // System.out.println(" force switch to fp=" + newFP + " oldFP=" + fp);- // }- nextEnt = -1;- fp = newFP;- } else {- // if (DEBUG) {- // System.out.println(" stay on same fp=" + newFP);- // }- }- }-- public void decodeMetaData() throws IOException {-- // if (DEBUG) System.out.println("\nBTTR.decodeMetadata seg=" + segment + " mdUpto=" +- // metaDataUpto + " vs termBlockOrd=" + state.termBlockOrd);-- // lazily catch up on metadata decode:- final int limit = getTermBlockOrd();- boolean absolute = metaDataUpto == 0;- assert limit > 0;-- // TODO: better API would be "jump straight to term=N"???- while (metaDataUpto < limit) {-- // TODO: we could make "tiers" of metadata, ie,- // decode docFreq/totalTF but don't decode postings- // metadata; this way caller could get- // docFreq/totalTF w/o paying decode cost for- // postings-- // TODO: if docFreq were bulk decoded we could- // just skipN here:-- if (version >= Lucene40BlockTreeTermsReader.VERSION_COMPRESSED_SUFFIXES) {- if (statsSingletonRunLength > 0) {- state.docFreq = 1;- state.totalTermFreq = 1;- statsSingletonRunLength--;- } else {- int token = statsReader.readVInt();- if ((token & 1) == 1) {- state.docFreq = 1;- state.totalTermFreq = 1;- statsSingletonRunLength = token >>> 1;- } else {- state.docFreq = token >>> 1;- if (ste.fr.fieldInfo.getIndexOptions() == IndexOptions.DOCS) {- state.totalTermFreq = state.docFreq;- } else {- state.totalTermFreq = state.docFreq + statsReader.readVLong();- }- }- }- } else {- assert statsSingletonRunLength == 0;- state.docFreq = statsReader.readVInt();- // if (DEBUG) System.out.println(" dF=" + state.docFreq);- if (ste.fr.fieldInfo.getIndexOptions() == IndexOptions.DOCS) {- state.totalTermFreq = state.docFreq; // all postings have freq=1- } else {- state.totalTermFreq = state.docFreq + statsReader.readVLong();- // if (DEBUG) System.out.println(" totTF=" + state.totalTermFreq);- }- }-- // metadata- ste.fr.parent.postingsReader.decodeTerm(bytesReader, ste.fr.fieldInfo, state, absolute);-- metaDataUpto++;- absolute = false;- }- state.termBlockOrd = metaDataUpto;- }-- // Used only by assert- private boolean prefixMatches(BytesRef target) {- for (int bytePos = 0; bytePos < prefix; bytePos++) {- if (target.bytes[target.offset + bytePos] != ste.term.byteAt(bytePos)) {- return false;- }- }-- return true;- }-- // Scans to sub-block that has this target fp; only- // called by next(); NOTE: does not set- // startBytePos/suffix as a side effect- public void scanToSubBlock(long subFP) {- assert !isLeafBlock;- // if (DEBUG) System.out.println(" scanToSubBlock fp=" + fp + " subFP=" + subFP + " entCount="- // + entCount + " lastSubFP=" + lastSubFP);- // assert nextEnt == 0;- if (lastSubFP == subFP) {- // if (DEBUG) System.out.println(" already positioned");- return;- }- assert subFP < fp : "fp=" + fp + " subFP=" + subFP;- final long targetSubCode = fp - subFP;- // if (DEBUG) System.out.println(" targetSubCode=" + targetSubCode);- while (true) {- assert nextEnt < entCount;- nextEnt++;- final int code = suffixLengthsReader.readVInt();- suffixesReader.skipBytes(code >>> 1);- if ((code & 1) != 0) {- final long subCode = suffixLengthsReader.readVLong();- if (targetSubCode == subCode) {- // if (DEBUG) System.out.println(" match!");- lastSubFP = subFP;- return;- }- } else {- state.termBlockOrd++;- }- }- }-- // NOTE: sets startBytePos/suffix as a side effect- public SeekStatus scanToTerm(BytesRef target, boolean exactOnly) throws IOException {- return isLeafBlock ? scanToTermLeaf(target, exactOnly) : scanToTermNonLeaf(target, exactOnly);- }-- private int startBytePos;- private int suffix;- private long subCode;- CompressionAlgorithm compressionAlg = CompressionAlgorithm.NO_COMPRESSION;-- // Target's prefix matches this block's prefix; we- // scan the entries check if the suffix matches.- public SeekStatus scanToTermLeaf(BytesRef target, boolean exactOnly) throws IOException {-- // if (DEBUG) System.out.println(" scanToTermLeaf: block fp=" + fp + " prefix=" + prefix +- // " nextEnt=" + nextEnt + " (of " + entCount + ") target=" +- // ToStringUtils.bytesRefToString(target) +- // " term=" + ToStringUtils.bytesRefToString(term));-- assert nextEnt != -1;-- ste.termExists = true;- subCode = 0;-- if (nextEnt == entCount) {- if (exactOnly) {- fillTerm();- }- return SeekStatus.END;- }-- assert prefixMatches(target);-- // TODO: binary search when all terms have the same length, which is common for ID fields,- // which are also the most sensitive to lookup performance?- // Loop over each entry (term or sub-block) in this block:- do {- nextEnt++;-- suffix = suffixLengthsReader.readVInt();-- // if (DEBUG) {- // BytesRef suffixBytesRef = new BytesRef();- // suffixBytesRef.bytes = suffixBytes;- // suffixBytesRef.offset = suffixesReader.getPosition();- // suffixBytesRef.length = suffix;- // System.out.println(" cycle: term " + (nextEnt-1) + " (of " + entCount + ") suffix="- // + ToStringUtils.bytesRefToString(suffixBytesRef));- // }-- startBytePos = suffixesReader.getPosition();- suffixesReader.skipBytes(suffix);-- // Loop over bytes in the suffix, comparing to the target- final int cmp =- Arrays.compareUnsigned(- suffixBytes,- startBytePos,- startBytePos + suffix,- target.bytes,- target.offset + prefix,- target.offset + target.length);-- if (cmp < 0) {- // Current entry is still before the target;- // keep scanning- } else if (cmp > 0) {- // Done! Current entry is after target --- // return NOT_FOUND:- fillTerm();-- // if (DEBUG) System.out.println(" not found");- return SeekStatus.NOT_FOUND;- } else {- // Exact match!-- // This cannot be a sub-block because we- // would have followed the index to this- // sub-block from the start:-- assert ste.termExists;- fillTerm();- // if (DEBUG) System.out.println(" found!");- return SeekStatus.FOUND;- }- } while (nextEnt < entCount);-- // It is possible (and OK) that terms index pointed us- // at this block, but, we scanned the entire block and- // did not find the term to position to. This happens- // when the target is after the last term in the block- // (but, before the next term in the index). EG- // target could be foozzz, and terms index pointed us- // to the foo* block, but the last term in this block- // was fooz (and, eg, first term in the next block will- // bee fop).- // if (DEBUG) System.out.println(" block end");- if (exactOnly) {- fillTerm();- }-- // TODO: not consistent that in the- // not-exact case we don't next() into the next- // frame here- return SeekStatus.END;- }-- // Target's prefix matches this block's prefix; we- // scan the entries check if the suffix matches.- public SeekStatus scanToTermNonLeaf(BytesRef target, boolean exactOnly) throws IOException {-- // if (DEBUG) System.out.println(" scanToTermNonLeaf: block fp=" + fp + " prefix=" + prefix +- // " nextEnt=" + nextEnt + " (of " + entCount + ") target=" +- // ToStringUtils.bytesRefToString(target) +- // " term=" + ToStringUtils.bytesRefToString(term));-- assert nextEnt != -1;-- if (nextEnt == entCount) {- if (exactOnly) {- fillTerm();- ste.termExists = subCode == 0;- }- return SeekStatus.END;- }-- assert prefixMatches(target);-- // Loop over each entry (term or sub-block) in this block:- while (nextEnt < entCount) {-- nextEnt++;-- final int code = suffixLengthsReader.readVInt();- suffix = code >>> 1;-- // if (DEBUG) {- // BytesRef suffixBytesRef = new BytesRef();- // suffixBytesRef.bytes = suffixBytes;- // suffixBytesRef.offset = suffixesReader.getPosition();- // suffixBytesRef.length = suffix;- // System.out.println(" cycle: " + ((code&1)==1 ? "sub-block" : "term") + " " +- // (nextEnt-1) + " (of " + entCount + ") suffix=" +- // ToStringUtils.bytesRefToString(suffixBytesRef));- // }-- final int termLen = prefix + suffix;- startBytePos = suffixesReader.getPosition();- suffixesReader.skipBytes(suffix);- ste.termExists = (code & 1) == 0;- if (ste.termExists) {- state.termBlockOrd++;- subCode = 0;- } else {- subCode = suffixLengthsReader.readVLong();- lastSubFP = fp - subCode;- }-- final int cmp =- Arrays.compareUnsigned(- suffixBytes,- startBytePos,- startBytePos + suffix,- target.bytes,- target.offset + prefix,- target.offset + target.length);-- if (cmp < 0) {- // Current entry is still before the target;- // keep scanning- } else if (cmp > 0) {- // Done! Current entry is after target --- // return NOT_FOUND:- fillTerm();-- // if (DEBUG) System.out.println(" maybe done exactOnly=" + exactOnly +- // " ste.termExists=" + ste.termExists);-- if (!exactOnly && !ste.termExists) {- // System.out.println(" now pushFrame");- // TODO this- // We are on a sub-block, and caller wants- // us to position to the next term after- // the target, so we must recurse into the- // sub-frame(s):- ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, termLen);- ste.currentFrame.loadBlock();- while (ste.currentFrame.next()) {- ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, ste.term.length());- ste.currentFrame.loadBlock();- }- }-- // if (DEBUG) System.out.println(" not found");- return SeekStatus.NOT_FOUND;- } else {- // Exact match!-- // This cannot be a sub-block because we- // would have followed the index to this- // sub-block from the start:-- assert ste.termExists;- fillTerm();- // if (DEBUG) System.out.println(" found!");- return SeekStatus.FOUND;- }- }-- // It is possible (and OK) that terms index pointed us- // at this block, but, we scanned the entire block and- // did not find the term to position to. This happens- // when the target is after the last term in the block- // (but, before the next term in the index). EG- // target could be foozzz, and terms index pointed us- // to the foo* block, but the last term in this block- // was fooz (and, eg, first term in the next block will- // bee fop).- // if (DEBUG) System.out.println(" block end");- if (exactOnly) {- fillTerm();- }-- // TODO: not consistent that in the- // not-exact case we don't next() into the next- // frame here- return SeekStatus.END;- }-- private void fillTerm() {- final int termLength = prefix + suffix;- ste.term.setLength(termLength);- ste.term.grow(termLength);- System.arraycopy(suffixBytes, startBytePos, ste.term.bytes(), prefix, suffix);- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/Stats.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/Stats.javadeleted file mode 100644index 812a959e714..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/Stats.java+++ /dev/null@@ -1,276 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene40.blocktree;--import static java.nio.charset.StandardCharsets.UTF_8;--import java.io.ByteArrayOutputStream;-import java.io.PrintStream;-import java.util.Locale;-import org.apache.lucene.codecs.PostingsReaderBase;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.BytesRef;--/**- * BlockTree statistics for a single field returned by {@link FieldReader#getStats()}.- *- * @lucene.internal- */-public class Stats {- /** Byte size of the index. */- public long indexNumBytes;-- /** Total number of terms in the field. */- public long totalTermCount;-- /** Total number of bytes (sum of term lengths) across all terms in the field. */- public long totalTermBytes;-- /** The number of normal (non-floor) blocks in the terms file. */- public int nonFloorBlockCount;-- /**- * The number of floor blocks (meta-blocks larger than the allowed {@code maxItemsPerBlock}) in- * the terms file.- */- public int floorBlockCount;-- /** The number of sub-blocks within the floor blocks. */- public int floorSubBlockCount;-- /** The number of "internal" blocks (that have both terms and sub-blocks). */- public int mixedBlockCount;-- /** The number of "leaf" blocks (blocks that have only terms). */- public int termsOnlyBlockCount;-- /** The number of "internal" blocks that do not contain terms (have only sub-blocks). */- public int subBlocksOnlyBlockCount;-- /** Total number of blocks. */- public int totalBlockCount;-- /** Number of blocks at each prefix depth. */- public int[] blockCountByPrefixLen = new int[10];-- private int startBlockCount;- private int endBlockCount;-- /** Total number of bytes used to store term suffixes. */- public long totalBlockSuffixBytes;-- /**- * Number of times each compression method has been used. 0 = uncompressed 1 = lowercase_ascii 2 =- * LZ4- */- public final long[] compressionAlgorithms = new long[3];-- /** Total number of suffix bytes before compression. */- public long totalUncompressedBlockSuffixBytes;-- /**- * Total number of bytes used to store term stats (not including what the {@link- * PostingsReaderBase} stores).- */- public long totalBlockStatsBytes;-- /**- * Total bytes stored by the {@link PostingsReaderBase}, plus the other few vInts stored in the- * frame.- */- public long totalBlockOtherBytes;-- /** Segment name. */- public final String segment;-- /** Field name. */- public final String field;-- Stats(String segment, String field) {- this.segment = segment;- this.field = field;- }-- void startBlock(SegmentTermsEnumFrame frame, boolean isFloor) {- totalBlockCount++;- if (isFloor) {- if (frame.fp == frame.fpOrig) {- floorBlockCount++;- }- floorSubBlockCount++;- } else {- nonFloorBlockCount++;- }-- if (blockCountByPrefixLen.length <= frame.prefix) {- blockCountByPrefixLen = ArrayUtil.grow(blockCountByPrefixLen, 1 + frame.prefix);- }- blockCountByPrefixLen[frame.prefix]++;- startBlockCount++;- totalBlockSuffixBytes += frame.totalSuffixBytes;- totalUncompressedBlockSuffixBytes += frame.suffixesReader.length();- if (frame.suffixesReader != frame.suffixLengthsReader) {- totalUncompressedBlockSuffixBytes += frame.suffixLengthsReader.length();- }- totalBlockStatsBytes += frame.statsReader.length();- compressionAlgorithms[frame.compressionAlg.code]++;- }-- void endBlock(SegmentTermsEnumFrame frame) {- final int termCount = frame.isLeafBlock ? frame.entCount : frame.state.termBlockOrd;- final int subBlockCount = frame.entCount - termCount;- totalTermCount += termCount;- if (termCount != 0 && subBlockCount != 0) {- mixedBlockCount++;- } else if (termCount != 0) {- termsOnlyBlockCount++;- } else if (subBlockCount != 0) {- subBlocksOnlyBlockCount++;- } else {- throw new IllegalStateException();- }- endBlockCount++;- final long otherBytes =- frame.fpEnd - frame.fp - frame.totalSuffixBytes - frame.statsReader.length();- assert otherBytes > 0- : "otherBytes=" + otherBytes + " frame.fp=" + frame.fp + " frame.fpEnd=" + frame.fpEnd;- totalBlockOtherBytes += otherBytes;- }-- void term(BytesRef term) {- totalTermBytes += term.length;- }-- void finish() {- assert startBlockCount == endBlockCount- : "startBlockCount=" + startBlockCount + " endBlockCount=" + endBlockCount;- assert totalBlockCount == floorSubBlockCount + nonFloorBlockCount- : "floorSubBlockCount="- + floorSubBlockCount- + " nonFloorBlockCount="- + nonFloorBlockCount- + " totalBlockCount="- + totalBlockCount;- assert totalBlockCount == mixedBlockCount + termsOnlyBlockCount + subBlocksOnlyBlockCount- : "totalBlockCount="- + totalBlockCount- + " mixedBlockCount="- + mixedBlockCount- + " subBlocksOnlyBlockCount="- + subBlocksOnlyBlockCount- + " termsOnlyBlockCount="- + termsOnlyBlockCount;- }-- @Override- public String toString() {- final ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);- PrintStream out = new PrintStream(bos, false, UTF_8);-- out.println(" index FST:");- out.println(" " + indexNumBytes + " bytes");- out.println(" terms:");- out.println(" " + totalTermCount + " terms");- out.println(- " "- + totalTermBytes- + " bytes"- + (totalTermCount != 0- ? " ("- + String.format(Locale.ROOT, "%.1f", ((double) totalTermBytes) / totalTermCount)- + " bytes/term)"- : ""));- out.println(" blocks:");- out.println(" " + totalBlockCount + " blocks");- out.println(" " + termsOnlyBlockCount + " terms-only blocks");- out.println(" " + subBlocksOnlyBlockCount + " sub-block-only blocks");- out.println(" " + mixedBlockCount + " mixed blocks");- out.println(" " + floorBlockCount + " floor blocks");- out.println(" " + (totalBlockCount - floorSubBlockCount) + " non-floor blocks");- out.println(" " + floorSubBlockCount + " floor sub-blocks");- out.println(- " "- + totalUncompressedBlockSuffixBytes- + " term suffix bytes before compression"- + (totalBlockCount != 0- ? " ("- + String.format(- Locale.ROOT, "%.1f", ((double) totalBlockSuffixBytes) / totalBlockCount)- + " suffix-bytes/block)"- : ""));- StringBuilder compressionCounts = new StringBuilder();- for (int code = 0; code < compressionAlgorithms.length; ++code) {- if (compressionAlgorithms[code] == 0) {- continue;- }- if (compressionCounts.length() > 0) {- compressionCounts.append(", ");- }- compressionCounts.append(CompressionAlgorithm.byCode(code));- compressionCounts.append(": ");- compressionCounts.append(compressionAlgorithms[code]);- }- out.println(- " "- + totalBlockSuffixBytes- + " compressed term suffix bytes"- + (totalBlockCount != 0- ? " ("- + String.format(- Locale.ROOT,- "%.2f",- ((double) totalBlockSuffixBytes) / totalUncompressedBlockSuffixBytes)- + " compression ratio - compression count by algorithm: "- + compressionCounts- : "")- + ")");- out.println(- " "- + totalBlockStatsBytes- + " term stats bytes "- + (totalBlockCount != 0- ? " ("- + String.format(- Locale.ROOT, "%.1f", ((double) totalBlockStatsBytes) / totalBlockCount)- + " stats-bytes/block)"- : ""));- out.println(- " "- + totalBlockOtherBytes- + " other bytes"- + (totalBlockCount != 0- ? " ("- + String.format(- Locale.ROOT, "%.1f", ((double) totalBlockOtherBytes) / totalBlockCount)- + " other-bytes/block)"- : ""));- if (totalBlockCount != 0) {- out.println(" by prefix length:");- int total = 0;- for (int prefix = 0; prefix < blockCountByPrefixLen.length; prefix++) {- final int blockCount = blockCountByPrefixLen[prefix];- total += blockCount;- if (blockCount != 0) {- out.println(" " + String.format(Locale.ROOT, "%2d", prefix) + ": " + blockCount);- }- }- assert totalBlockCount == total;- }-- return bos.toString(UTF_8);- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/package-info.javadeleted file mode 100644index a34b7aa345d..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/package-info.java+++ /dev/null@@ -1,28 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--/**- * BlockTree terms dictionary.- *- * <p>This terms dictionary organizes all terms into blocks according to shared prefix, such that- * each block has enough terms, and then stores the prefix trie in memory as an FST as the index- * structure. It allows you to plug in your own {@link org.apache.lucene.codecs.PostingsWriterBase}- * to implement the postings.- *- * <p>See {@code BlockTreeTermsWriter} for the file format.- */-package org.apache.lucene.backward_codecs.lucene40.blocktree;diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/ForUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/ForUtil.javadeleted file mode 100644index b6131049694..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/ForUtil.java+++ /dev/null@@ -1,239 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.BLOCK_SIZE;--import java.io.IOException;-import java.util.Arrays;-import org.apache.lucene.store.DataInput;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.store.IndexOutput;-import org.apache.lucene.util.packed.PackedInts;-import org.apache.lucene.util.packed.PackedInts.Decoder;-import org.apache.lucene.util.packed.PackedInts.FormatAndBits;--/**- * Encode all values in normal area with fixed bit width, which is determined by the max value in- * this block.- */-final class ForUtil {-- /** Special number of bits per value used whenever all values to encode are equal. */- private static final int ALL_VALUES_EQUAL = 0;-- /**- * Upper limit of the number of bytes that might be required to stored <code>BLOCK_SIZE</code>- * encoded values.- */- static final int MAX_ENCODED_SIZE = BLOCK_SIZE * 4;-- /**- * Upper limit of the number of values that might be decoded in a single call to {@link- * #readBlock(IndexInput, byte[], int[])}. Although values after <code>BLOCK_SIZE</code> are- * garbage, it is necessary to allocate value buffers whose size is {@code >= MAX_DATA_SIZE} to- * avoid {@link ArrayIndexOutOfBoundsException}s.- */- static final int MAX_DATA_SIZE;-- static {- int maxDataSize = 0;- for (int version = PackedInts.VERSION_START; version <= PackedInts.VERSION_CURRENT; version++) {- for (PackedInts.Format format : PackedInts.Format.values()) {- for (int bpv = 1; bpv <= 32; ++bpv) {- if (!format.isSupported(bpv)) {- continue;- }- final PackedInts.Decoder decoder = PackedInts.getDecoder(format, version, bpv);- final int iterations = computeIterations(decoder);- maxDataSize = Math.max(maxDataSize, iterations * decoder.byteValueCount());- }- }- }- MAX_DATA_SIZE = maxDataSize;- }-- /**- * Compute the number of iterations required to decode <code>BLOCK_SIZE</code> values with the- * provided {@link Decoder}.- */- private static int computeIterations(PackedInts.Decoder decoder) {- return (int) Math.ceil((float) BLOCK_SIZE / decoder.byteValueCount());- }-- /**- * Compute the number of bytes required to encode a block of values that require <code>- * bitsPerValue</code> bits per value with format <code>format</code>.- */- private static int encodedSize(- PackedInts.Format format, int packedIntsVersion, int bitsPerValue) {- final long byteCount = format.byteCount(packedIntsVersion, BLOCK_SIZE, bitsPerValue);- assert byteCount >= 0 && byteCount <= Integer.MAX_VALUE : byteCount;- return (int) byteCount;- }-- private final int[] encodedSizes;- private final PackedInts.Encoder[] encoders;- private final PackedInts.Decoder[] decoders;- private final int[] iterations;-- /** Create a new {@link ForUtil} instance and save state into <code>out</code>. */- ForUtil(float acceptableOverheadRatio, DataOutput out) throws IOException {- out.writeVInt(PackedInts.VERSION_CURRENT);- encodedSizes = new int[33];- encoders = new PackedInts.Encoder[33];- decoders = new PackedInts.Decoder[33];- iterations = new int[33];-- for (int bpv = 1; bpv <= 32; ++bpv) {- final FormatAndBits formatAndBits =- PackedInts.fastestFormatAndBits(BLOCK_SIZE, bpv, acceptableOverheadRatio);- assert formatAndBits.format().isSupported(formatAndBits.bitsPerValue());- assert formatAndBits.bitsPerValue() <= 32;- encodedSizes[bpv] =- encodedSize(- formatAndBits.format(), PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue());- encoders[bpv] =- PackedInts.getEncoder(- formatAndBits.format(), PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue());- decoders[bpv] =- PackedInts.getDecoder(- formatAndBits.format(), PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue());- iterations[bpv] = computeIterations(decoders[bpv]);-- out.writeVInt(formatAndBits.format().getId() << 5 | (formatAndBits.bitsPerValue() - 1));- }- }-- /** Restore a {@link ForUtil} from a {@link DataInput}. */- ForUtil(DataInput in) throws IOException {- int packedIntsVersion = in.readVInt();- PackedInts.checkVersion(packedIntsVersion);- encodedSizes = new int[33];- encoders = new PackedInts.Encoder[33];- decoders = new PackedInts.Decoder[33];- iterations = new int[33];-- for (int bpv = 1; bpv <= 32; ++bpv) {- final int code = in.readVInt();- final int formatId = code >>> 5;- final int bitsPerValue = (code & 31) + 1;-- final PackedInts.Format format = PackedInts.Format.byId(formatId);- assert format.isSupported(bitsPerValue);- encodedSizes[bpv] = encodedSize(format, packedIntsVersion, bitsPerValue);- encoders[bpv] = PackedInts.getEncoder(format, packedIntsVersion, bitsPerValue);- decoders[bpv] = PackedInts.getDecoder(format, packedIntsVersion, bitsPerValue);- iterations[bpv] = computeIterations(decoders[bpv]);- }- }-- /**- * Write a block of data (<code>For</code> format).- *- * @param data the data to write- * @param encoded a buffer to use to encode data- * @param out the destination output- * @throws IOException If there is a low-level I/O error- */- void writeBlock(int[] data, byte[] encoded, IndexOutput out) throws IOException {- if (isAllEqual(data)) {- out.writeByte((byte) ALL_VALUES_EQUAL);- out.writeVInt(data[0]);- return;- }-- final int numBits = bitsRequired(data);- assert numBits > 0 && numBits <= 32 : numBits;- final PackedInts.Encoder encoder = encoders[numBits];- final int iters = iterations[numBits];- assert iters * encoder.byteValueCount() >= BLOCK_SIZE;- final int encodedSize = encodedSizes[numBits];- assert iters * encoder.byteBlockCount() >= encodedSize;-- out.writeByte((byte) numBits);-- encoder.encode(data, 0, encoded, 0, iters);- out.writeBytes(encoded, encodedSize);- }-- /**- * Read the next block of data (<code>For</code> format).- *- * @param in the input to use to read data- * @param encoded a buffer that can be used to store encoded data- * @param decoded where to write decoded data- * @throws IOException If there is a low-level I/O error- */- void readBlock(IndexInput in, byte[] encoded, int[] decoded) throws IOException {- final int numBits = in.readByte();- assert numBits <= 32 : numBits;-- if (numBits == ALL_VALUES_EQUAL) {- final int value = in.readVInt();- Arrays.fill(decoded, 0, BLOCK_SIZE, value);- return;- }-- final int encodedSize = encodedSizes[numBits];- in.readBytes(encoded, 0, encodedSize);-- final PackedInts.Decoder decoder = decoders[numBits];- final int iters = iterations[numBits];- assert iters * decoder.byteValueCount() >= BLOCK_SIZE;-- decoder.decode(encoded, 0, decoded, 0, iters);- }-- /**- * Skip the next block of data.- *- * @param in the input where to read data- * @throws IOException If there is a low-level I/O error- */- void skipBlock(IndexInput in) throws IOException {- final int numBits = in.readByte();- if (numBits == ALL_VALUES_EQUAL) {- in.readVInt();- return;- }- assert numBits > 0 && numBits <= 32 : numBits;- final int encodedSize = encodedSizes[numBits];- in.seek(in.getFilePointer() + encodedSize);- }-- private static boolean isAllEqual(final int[] data) {- final int v = data[0];- for (int i = 1; i < BLOCK_SIZE; ++i) {- if (data[i] != v) {- return false;- }- }- return true;- }-- /** Compute the number of bits required to serialize any of the longs in <code>data</code>. */- private static int bitsRequired(final int[] data) {- long or = 0;- for (int i = 0; i < BLOCK_SIZE; ++i) {- assert data[i] >= 0;- or |= data[i];- }- return PackedInts.bitsRequired(or);- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50CompoundFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50CompoundFormat.javadeleted file mode 100644index ca697b2948b..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50CompoundFormat.java+++ /dev/null@@ -1,88 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import java.io.IOException;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.CompoundDirectory;-import org.apache.lucene.codecs.CompoundFormat;-import org.apache.lucene.index.SegmentInfo;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;--/**- * Lucene 5.0 compound file format- *- * <p>Files:- *- * <ul>- * <li><code>.cfs</code>: An optional "virtual" file consisting of all the other index files for- * systems that frequently run out of file handles.- * <li><code>.cfe</code>: The "virtual" compound file's entry table holding all entries in the- * corresponding .cfs file.- * </ul>- *- * <p>Description:- *- * <ul>- * <li>Compound (.cfs) --> Header, FileData <sup>FileCount</sup>, Footer- * <li>Compound Entry Table (.cfe) --> Header, FileCount, <FileName, DataOffset,- * DataLength> <sup>FileCount</sup>- * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}- * <li>FileCount --> {@link DataOutput#writeVInt VInt}- * <li>DataOffset,DataLength,Checksum --> {@link DataOutput#writeLong UInt64}- * <li>FileName --> {@link DataOutput#writeString String}- * <li>FileData --> raw file data- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}- * </ul>- *- * <p>Notes:- *- * <ul>- * <li>FileCount indicates how many files are contained in this compound file. The entry table- * that follows has that many entries.- * <li>Each directory entry contains a long pointer to the start of this file's data section, the- * files length, and a String with that file's name.- * </ul>- */-public final class Lucene50CompoundFormat extends CompoundFormat {-- /** Extension of compound file */- static final String DATA_EXTENSION = "cfs";-- /** Extension of compound file entries */- static final String ENTRIES_EXTENSION = "cfe";-- static final String DATA_CODEC = "Lucene50CompoundData";- static final String ENTRY_CODEC = "Lucene50CompoundEntries";- static final int VERSION_START = 0;- static final int VERSION_CURRENT = VERSION_START;-- /** Sole constructor. */- public Lucene50CompoundFormat() {}-- @Override- public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si) throws IOException {- return new Lucene50CompoundReader(dir, si);- }-- @Override- public void write(Directory dir, SegmentInfo si, IOContext context) throws IOException {- throw new UnsupportedOperationException("Old formats can't be used for writing");- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50CompoundReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50CompoundReader.javadeleted file mode 100644index 819d45a5328..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50CompoundReader.java+++ /dev/null@@ -1,215 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import java.io.FileNotFoundException;-import java.io.IOException;-import java.util.Collections;-import java.util.HashMap;-import java.util.Map;-import java.util.Set;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.CompoundDirectory;-import org.apache.lucene.index.CorruptIndexException;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.SegmentInfo;-import org.apache.lucene.store.ChecksumIndexInput;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.util.IOUtils;--/**- * Class for accessing a compound stream. This class implements a directory, but is limited to only- * read operations. Directory methods that would normally modify data throw an exception.- *- * @lucene.experimental- */-final class Lucene50CompoundReader extends CompoundDirectory {-- /** Offset/Length for a slice inside of a compound file */- public static final class FileEntry {- long offset;- long length;- }-- private final Directory directory;- private final String segmentName;- private final Map<String, FileEntry> entries;- private final IndexInput handle;- private int version;-- /** Create a new CompoundFileDirectory. */- // TODO: we should just pre-strip "entries" and append segment name up-front like simpletext?- // this need not be a "general purpose" directory anymore (it only writes index files)- public Lucene50CompoundReader(Directory directory, SegmentInfo si) throws IOException {- this.directory = directory;- this.segmentName = si.name;- String dataFileName =- IndexFileNames.segmentFileName(segmentName, "", Lucene50CompoundFormat.DATA_EXTENSION);- String entriesFileName =- IndexFileNames.segmentFileName(segmentName, "", Lucene50CompoundFormat.ENTRIES_EXTENSION);- this.entries = readEntries(si.getId(), directory, entriesFileName);- boolean success = false;-- long expectedLength = CodecUtil.indexHeaderLength(Lucene50CompoundFormat.DATA_CODEC, "");- for (Map.Entry<String, FileEntry> ent : entries.entrySet()) {- expectedLength += ent.getValue().length;- }- expectedLength += CodecUtil.footerLength();-- handle = directory.openInput(dataFileName, IOContext.DEFAULT);- // DirectoryUtil.openInput(directory, dataFileName, context);- try {- CodecUtil.checkIndexHeader(- handle, Lucene50CompoundFormat.DATA_CODEC, version, version, si.getId(), "");-- // NOTE: data file is too costly to verify checksum against all the bytes on open,- // but for now we at least verify proper structure of the checksum footer: which looks- // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption- // such as file truncation.- CodecUtil.retrieveChecksum(handle);-- // We also validate length, because e.g. if you strip 16 bytes off the .cfs we otherwise- // would not detect it:- if (handle.length() != expectedLength) {- throw new CorruptIndexException(- "length should be " + expectedLength + " bytes, but is " + handle.length() + " instead",- handle);- }-- success = true;- } finally {- if (!success) {- IOUtils.closeWhileHandlingException(handle);- }- }- }-- /** Helper method that reads CFS entries from an input stream */- private Map<String, FileEntry> readEntries(- byte[] segmentID, Directory dir, String entriesFileName) throws IOException {- Map<String, FileEntry> mapping = null;- try (ChecksumIndexInput entriesStream =- EndiannessReverserUtil.openChecksumInput(dir, entriesFileName, IOContext.READONCE)) {- Throwable priorE = null;- try {- version =- CodecUtil.checkIndexHeader(- entriesStream,- Lucene50CompoundFormat.ENTRY_CODEC,- Lucene50CompoundFormat.VERSION_START,- Lucene50CompoundFormat.VERSION_CURRENT,- segmentID,- "");-- mapping = readMapping(entriesStream);-- } catch (Throwable exception) {- priorE = exception;- } finally {- CodecUtil.checkFooter(entriesStream, priorE);- }- }- return Collections.unmodifiableMap(mapping);- }-- private Map<String, FileEntry> readMapping(IndexInput entriesStream) throws IOException {- final int numEntries = entriesStream.readVInt();- Map<String, FileEntry> mapping = HashMap.newHashMap(numEntries);- for (int i = 0; i < numEntries; i++) {- final FileEntry fileEntry = new FileEntry();- final String id = entriesStream.readString();- FileEntry previous = mapping.put(id, fileEntry);- if (previous != null) {- throw new CorruptIndexException("Duplicate cfs entry id=" + id + " in CFS ", entriesStream);- }- fileEntry.offset = entriesStream.readLong();- fileEntry.length = entriesStream.readLong();- }- return mapping;- }-- @Override- public void close() throws IOException {- IOUtils.close(handle);- }-- @Override- public IndexInput openInput(String name, IOContext context) throws IOException {- ensureOpen();- final String id = IndexFileNames.stripSegmentName(name);- final FileEntry entry = entries.get(id);- if (entry == null) {- String datFileName =- IndexFileNames.segmentFileName(segmentName, "", Lucene50CompoundFormat.DATA_EXTENSION);- throw new FileNotFoundException(- "No sub-file with id "- + id- + " found in compound file \""- + datFileName- + "\" (fileName="- + name- + " files: "- + entries.keySet()- + ")");- }- return handle.slice(name, entry.offset, entry.length, context);- }-- /** Returns an array of strings, one for each file in the directory. */- @Override- public String[] listAll() {- ensureOpen();- String[] res = entries.keySet().toArray(new String[entries.size()]);-- // Add the segment name- for (int i = 0; i < res.length; i++) {- res[i] = segmentName + res[i];- }- return res;- }-- /**- * Returns the length of a file in the directory.- *- * @throws IOException if the file does not exist- */- @Override- public long fileLength(String name) throws IOException {- ensureOpen();- FileEntry e = entries.get(IndexFileNames.stripSegmentName(name));- if (e == null) throw new FileNotFoundException(name);- return e.length;- }-- @Override- public String toString() {- return "CompoundFileDirectory(segment=\"" + segmentName + "\" in dir=" + directory + ")";- }-- @Override- public Set<String> getPendingDeletions() {- return Collections.emptySet();- }-- @Override- public void checkIntegrity() throws IOException {- CodecUtil.checksumEntireFile(handle);- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50LiveDocsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50LiveDocsFormat.javadeleted file mode 100644index b0187975742..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50LiveDocsFormat.java+++ /dev/null@@ -1,171 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import java.io.IOException;-import java.util.Collection;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.LiveDocsFormat;-import org.apache.lucene.index.CorruptIndexException;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.SegmentCommitInfo;-import org.apache.lucene.store.ChecksumIndexInput;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.store.IndexOutput;-import org.apache.lucene.util.Bits;-import org.apache.lucene.util.FixedBitSet;--/**- * Lucene 5.0 live docs format- *- * <p>The .liv file is optional, and only exists when a segment contains deletions.- *- * <p>Although per-segment, this file is maintained exterior to compound segment files.- *- * <p>Deletions (.liv) --> IndexHeader,Generation,Bits- *- * <ul>- * <li>SegmentHeader --> {@link CodecUtil#writeIndexHeader IndexHeader}- * <li>Bits --> <{@link DataOutput#writeLong Int64}> <sup>LongCount</sup>- * </ul>- */-public final class Lucene50LiveDocsFormat extends LiveDocsFormat {-- /** extension of live docs */- private static final String EXTENSION = "liv";-- /** codec of live docs */- private static final String CODEC_NAME = "Lucene50LiveDocs";-- /** supported version range */- private static final int VERSION_START = 0;-- private static final int VERSION_CURRENT = VERSION_START;-- /** Sole constructor. */- public Lucene50LiveDocsFormat() {}-- @Override- public Bits readLiveDocs(Directory dir, SegmentCommitInfo info, IOContext context)- throws IOException {- long gen = info.getDelGen();- String name = IndexFileNames.fileNameFromGeneration(info.info.name, EXTENSION, gen);- final int length = info.info.maxDoc();- try (ChecksumIndexInput input = EndiannessReverserUtil.openChecksumInput(dir, name, context)) {- Throwable priorE = null;- try {- CodecUtil.checkIndexHeader(- input,- CODEC_NAME,- VERSION_START,- VERSION_CURRENT,- info.info.getId(),- Long.toString(gen, Character.MAX_RADIX));-- FixedBitSet fbs = readFixedBitSet(input, length);-- if (fbs.length() - fbs.cardinality() != info.getDelCount()) {- throw new CorruptIndexException(- "bits.deleted="- + (fbs.length() - fbs.cardinality())- + " info.delcount="- + info.getDelCount(),- input);- }- return fbs.asReadOnlyBits();- } catch (Throwable exception) {- priorE = exception;- } finally {- CodecUtil.checkFooter(input, priorE);- }- }- throw new AssertionError();- }-- private FixedBitSet readFixedBitSet(IndexInput input, int length) throws IOException {- long[] data = new long[FixedBitSet.bits2words(length)];- for (int i = 0; i < data.length; i++) {- data[i] = input.readLong();- }- return new FixedBitSet(data, length);- }-- /**- * Note: although this format is only used on older versions, we need to keep the write logic in- * addition to the read logic. When we delete documents that live in an older segment, we write to- * the live docs for that segment.- */- @Override- public void writeLiveDocs(- Bits bits, Directory dir, SegmentCommitInfo info, int newDelCount, IOContext context)- throws IOException {- long gen = info.getNextDelGen();- String name = IndexFileNames.fileNameFromGeneration(info.info.name, EXTENSION, gen);- int delCount;- try (IndexOutput output = EndiannessReverserUtil.createOutput(dir, name, context)) {-- CodecUtil.writeIndexHeader(- output,- CODEC_NAME,- VERSION_CURRENT,- info.info.getId(),- Long.toString(gen, Character.MAX_RADIX));-- delCount = writeBits(output, bits);-- CodecUtil.writeFooter(output);- }- if (delCount != info.getDelCount() + newDelCount) {- throw new CorruptIndexException(- "bits.deleted="- + delCount- + " info.delcount="- + info.getDelCount()- + " newdelcount="- + newDelCount,- name);- }- }-- private int writeBits(IndexOutput output, Bits bits) throws IOException {- int delCount = 0;- final int longCount = FixedBitSet.bits2words(bits.length());- for (int i = 0; i < longCount; ++i) {- long currentBits = 0;- for (int j = i << 6, end = Math.min(j + 63, bits.length() - 1); j <= end; ++j) {- if (bits.get(j)) {- currentBits |= 1L << j; // mod 64- } else {- delCount += 1;- }- }- output.writeLong(currentBits);- }- return delCount;- }-- @Override- public void files(SegmentCommitInfo info, Collection<String> files) throws IOException {- if (info.hasDeletions()) {- files.add(IndexFileNames.fileNameFromGeneration(info.info.name, EXTENSION, info.getDelGen()));- }- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50PostingsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50PostingsFormat.javadeleted file mode 100644index 1d08136cb99..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50PostingsFormat.java+++ /dev/null@@ -1,474 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import java.io.IOException;-import org.apache.lucene.backward_codecs.lucene40.blocktree.Lucene40BlockTreeTermsReader;-import org.apache.lucene.codecs.BlockTermState;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.FieldsConsumer;-import org.apache.lucene.codecs.FieldsProducer;-import org.apache.lucene.codecs.MultiLevelSkipListWriter;-import org.apache.lucene.codecs.PostingsFormat;-import org.apache.lucene.codecs.PostingsReaderBase;-import org.apache.lucene.index.IndexOptions;-import org.apache.lucene.index.SegmentReadState;-import org.apache.lucene.index.SegmentWriteState;-import org.apache.lucene.index.TermState;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.util.IOUtils;-import org.apache.lucene.util.packed.PackedInts;--/**- * Lucene 5.0 postings format, which encodes postings in packed integer blocks for fast decode.- *- * <p>Basic idea:- *- * <ul>- * <li><b>Packed Blocks and VInt Blocks</b>:- * <p>In packed blocks, integers are encoded with the same bit width ({@link PackedInts packed- * format}): the block size (i.e. number of integers inside block) is fixed (currently 128).- * Additionally blocks that are all the same value are encoded in an optimized way.- * <p>In VInt blocks, integers are encoded as {@link DataOutput#writeVInt VInt}: the block- * size is variable.- * <li><b>Block structure</b>:- * <p>When the postings are long enough, Lucene50PostingsFormat will try to encode most- * integer data as a packed block.- * <p>Take a term with 259 documents as an example, the first 256 document ids are encoded as- * two packed blocks, while the remaining 3 are encoded as one VInt block.- * <p>Different kinds of data are always encoded separately into different packed blocks, but- * may possibly be interleaved into the same VInt block.- * <p>This strategy is applied to pairs: <document number, frequency>, <position,- * payload length>, <position, offset start, offset length>, and <position,- * payload length, offsetstart, offset length>.- * <li><b>Skipdata settings</b>:- * <p>The structure of skip table is quite similar to previous version of Lucene. Skip- * interval is the same as block size, and each skip entry points to the beginning of each- * block. However, for the first block, skip data is omitted.- * <li><b>Positions, Payloads, and Offsets</b>:- * <p>A position is an integer indicating where the term occurs within one document. A payload- * is a blob of metadata associated with current position. An offset is a pair of integers- * indicating the tokenized start/end offsets for given term in current position: it is- * essentially a specialized payload.- * <p>When payloads and offsets are not omitted, numPositions==numPayloads==numOffsets- * (assuming a null payload contributes one count). As mentioned in block structure, it is- * possible to encode these three either combined or separately.- * <p>In all cases, payloads and offsets are stored together. When encoded as a packed block,- * position data is separated out as .pos, while payloads and offsets are encoded in .pay- * (payload metadata will also be stored directly in .pay). When encoded as VInt blocks, all- * these three are stored interleaved into the .pos (so is payload metadata).- * <p>With this strategy, the majority of payload and offset data will be outside .pos file.- * So for queries that require only position data, running on a full index with payloads and- * offsets, this reduces disk pre-fetches.- * </ul>- *- * <p>Files and detailed format:- *- * <ul>- * <li><code>.tim</code>: <a href="#Termdictionary">Term Dictionary</a>- * <li><code>.tip</code>: <a href="#Termindex">Term Index</a>- * <li><code>.doc</code>: <a href="#Frequencies">Frequencies and Skip Data</a>- * <li><code>.pos</code>: <a href="#Positions">Positions</a>- * <li><code>.pay</code>: <a href="#Payloads">Payloads and Offsets</a>- * </ul>- *- * <a id="Termdictionary"></a>- *- * <dl>- * <dd><b>Term Dictionary</b>- * <p>The .tim file contains the list of terms in each field along with per-term statistics- * (such as docfreq) and pointers to the frequencies, positions, payload and skip data in the- * .doc, .pos, and .pay files. See {@code BlockTreeTermsWriter} for more details on the- * format.- * <p>NOTE: The term dictionary can plug into different postings implementations: the postings- * writer/reader are actually responsible for encoding and decoding the PostingsHeader and- * TermMetadata sections described here:- * <ul>- * <li>PostingsHeader --> Header, PackedBlockSize- * <li>TermMetadata --> (DocFPDelta|SingletonDocID), PosFPDelta?, PosVIntBlockFPDelta?,- * PayFPDelta?, SkipFPDelta?- * <li>Header, --> {@link CodecUtil#writeIndexHeader IndexHeader}- * <li>PackedBlockSize, SingletonDocID --> {@link DataOutput#writeVInt VInt}- * <li>DocFPDelta, PosFPDelta, PayFPDelta, PosVIntBlockFPDelta, SkipFPDelta --> {@link- * DataOutput#writeVLong VLong}- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}- * </ul>- * <p>Notes:- * <ul>- * <li>Header is a {@link CodecUtil#writeIndexHeader IndexHeader} storing the version- * information for the postings.- * <li>PackedBlockSize is the fixed block size for packed blocks. In packed block, bit width- * is determined by the largest integer. Smaller block size result in smaller variance- * among width of integers hence smaller indexes. Larger block size result in more- * efficient bulk i/o hence better acceleration. This value should always be a multiple- * of 64, currently fixed as 128 as a tradeoff. It is also the skip interval used to- * accelerate {@link org.apache.lucene.index.PostingsEnum#advance(int)}.- * <li>DocFPDelta determines the position of this term's TermFreqs within the .doc file. In- * particular, it is the difference of file offset between this term's data and previous- * term's data (or zero, for the first term in the block).On disk it is stored as the- * difference from previous value in sequence.- * <li>PosFPDelta determines the position of this term's TermPositions within the .pos file.- * While PayFPDelta determines the position of this term's <TermPayloads,- * TermOffsets?> within the .pay file. Similar to DocFPDelta, it is the difference- * between two file positions (or neglected, for fields that omit payloads and offsets).- * <li>PosVIntBlockFPDelta determines the position of this term's last TermPosition in last- * pos packed block within the .pos file. It is synonym for PayVIntBlockFPDelta or- * OffsetVIntBlockFPDelta. This is actually used to indicate whether it is necessary to- * load following payloads and offsets from .pos instead of .pay. Every time a new block- * of positions are to be loaded, the PostingsReader will use this value to check- * whether current block is packed format or VInt. When packed format, payloads and- * offsets are fetched from .pay, otherwise from .pos. (this value is neglected when- * total number of positions i.e. totalTermFreq is less or equal to PackedBlockSize).- * <li>SkipFPDelta determines the position of this term's SkipData within the .doc file. In- * particular, it is the length of the TermFreq data. SkipDelta is only stored if- * DocFreq is not smaller than SkipMinimum (i.e. 128 in Lucene50PostingsFormat).- * <li>SingletonDocID is an optimization when a term only appears in one document. In this- * case, instead of writing a file pointer to the .doc file (DocFPDelta), and then a- * VIntBlock at that location, the single document ID is written to the term dictionary.- * </ul>- * </dl>- *- * <a id="Termindex"></a>- *- * <dl>- * <dd><b>Term Index</b>- * <p>The .tip file contains an index into the term dictionary, so that it can be accessed- * randomly. See {@code BlockTreeTermsWriter} for more details on the format.- * </dl>- *- * <a id="Frequencies"></a>- *- * <dl>- * <dd><b>Frequencies and Skip Data</b>- * <p>The .doc file contains the lists of documents which contain each term, along with the- * frequency of the term in that document (except when frequencies are omitted: {@link- * IndexOptions#DOCS}). It also saves skip data to the beginning of each packed or VInt block,- * when the length of document list is larger than packed block size.- * <ul>- * <li>docFile(.doc) --> Header, <TermFreqs, SkipData?><sup>TermCount</sup>, Footer- * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}- * <li>TermFreqs --> <PackedBlock> <sup>PackedDocBlockNum</sup>, VIntBlock?- * <li>PackedBlock --> PackedDocDeltaBlock, PackedFreqBlock?- * <li>VIntBlock --> <DocDelta[,- * Freq?]><sup>DocFreq-PackedBlockSize*PackedDocBlockNum</sup>- * <li>SkipData --> <<SkipLevelLength, SkipLevel> <sup>NumSkipLevels-1</sup>,- * SkipLevel>, SkipDatum?- * <li>SkipLevel --> <SkipDatum> <sup>TrimmedDocFreq/(PackedBlockSize^(Level +- * 1))</sup>- * <li>SkipDatum --> DocSkip, DocFPSkip, <PosFPSkip, PosBlockOffset, PayLength?,- * PayFPSkip?>?, SkipChildLevelPointer?- * <li>PackedDocDeltaBlock, PackedFreqBlock --> {@link PackedInts PackedInts}- * <li>DocDelta, Freq, DocSkip, DocFPSkip, PosFPSkip, PosBlockOffset, PayByteUpto, PayFPSkip- * --> {@link DataOutput#writeVInt VInt}- * <li>SkipChildLevelPointer --> {@link DataOutput#writeVLong VLong}- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}- * </ul>- * <p>Notes:- * <ul>- * <li>PackedDocDeltaBlock is theoretically generated from two steps:- * <ol>- * <li>Calculate the difference between each document number and previous one, and get- * a d-gaps list (for the first document, use absolute value);- * <li>For those d-gaps from first one to- * PackedDocBlockNum*PackedBlockSize<sup>th</sup>, separately encode as packed- * blocks.- * </ol>- * If frequencies are not omitted, PackedFreqBlock will be generated without d-gap step.- * <li>VIntBlock stores remaining d-gaps (along with frequencies when possible) with a- * format that encodes DocDelta and Freq:- * <p>DocDelta: if frequencies are indexed, this determines both the document number and- * the frequency. In particular, DocDelta/2 is the difference between this document- * number and the previous document number (or zero when this is the first document in a- * TermFreqs). When DocDelta is odd, the frequency is one. When DocDelta is even, the- * frequency is read as another VInt. If frequencies are omitted, DocDelta contains the- * gap (not multiplied by 2) between document numbers and no frequency information is- * stored.- * <p>For example, the TermFreqs for a term which occurs once in document seven and- * three times in document eleven, with frequencies indexed, would be the following- * sequence of VInts:- * <p>15, 8, 3- * <p>If frequencies were omitted ({@link IndexOptions#DOCS}) it would be this sequence- * of VInts instead:- * <p>7,4- * <li>PackedDocBlockNum is the number of packed blocks for current term's docids or- * frequencies. In particular, PackedDocBlockNum = floor(DocFreq/PackedBlockSize)- * <li>TrimmedDocFreq = DocFreq % PackedBlockSize == 0 ? DocFreq - 1 : DocFreq. We use this- * trick since the definition of skip entry is a little different from base interface.- * In {@link MultiLevelSkipListWriter}, skip data is assumed to be saved for- * skipInterval<sup>th</sup>, 2*skipInterval<sup>th</sup> ... posting in the list.- * However, in Lucene50PostingsFormat, the skip data is saved for- * skipInterval+1<sup>th</sup>, 2*skipInterval+1<sup>th</sup> ... posting- * (skipInterval==PackedBlockSize in this case). When DocFreq is multiple of- * PackedBlockSize, MultiLevelSkipListWriter will expect one more skip data than- * Lucene50SkipWriter.- * <li>SkipDatum is the metadata of one skip entry. For the first block (no matter packed or- * VInt), it is omitted.- * <li>DocSkip records the document number of every PackedBlockSize<sup>th</sup> document- * number in the postings (i.e. last document number in each packed block). On disk it- * is stored as the difference from previous value in the sequence.- * <li>DocFPSkip records the file offsets of each block (excluding )posting at- * PackedBlockSize+1<sup>th</sup>, 2*PackedBlockSize+1<sup>th</sup> ... , in DocFile.- * The file offsets are relative to the start of current term's TermFreqs. On disk it is- * also stored as the difference from previous SkipDatum in the sequence.- * <li>Since positions and payloads are also block encoded, the skip should skip to related- * block first, then fetch the values according to in-block offset. PosFPSkip and- * PayFPSkip record the file offsets of related block in .pos and .pay, respectively.- * While PosBlockOffset indicates which value to fetch inside the related block- * (PayBlockOffset is unnecessary since it is always equal to PosBlockOffset). Same as- * DocFPSkip, the file offsets are relative to the start of current term's TermFreqs,- * and stored as a difference sequence.- * <li>PayByteUpto indicates the start offset of the current payload. It is equivalent to- * the sum of the payload lengths in the current block up to PosBlockOffset- * </ul>- * </dl>- *- * <a id="Positions"></a>- *- * <dl>- * <dd><b>Positions</b>- * <p>The .pos file contains the lists of positions that each term occurs at within documents.- * It also sometimes stores part of payloads and offsets for speedup.- * <ul>- * <li>PosFile(.pos) --> Header, <TermPositions> <sup>TermCount</sup>, Footer- * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}- * <li>TermPositions --> <PackedPosDeltaBlock> <sup>PackedPosBlockNum</sup>,- * VIntBlock?- * <li>VIntBlock --> <PositionDelta[, PayloadLength?], PayloadData?, OffsetDelta?,- * OffsetLength?><sup>PosVIntCount</sup>- * <li>PackedPosDeltaBlock --> {@link PackedInts PackedInts}- * <li>PositionDelta, OffsetDelta, OffsetLength --> {@link DataOutput#writeVInt VInt}- * <li>PayloadData --> {@link DataOutput#writeByte byte}<sup>PayLength</sup>- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}- * </ul>- * <p>Notes:- * <ul>- * <li>TermPositions are order by term (terms are implicit, from the term dictionary), and- * position values for each term document pair are incremental, and ordered by document- * number.- * <li>PackedPosBlockNum is the number of packed blocks for current term's positions,- * payloads or offsets. In particular, PackedPosBlockNum =- * floor(totalTermFreq/PackedBlockSize)- * <li>PosVIntCount is the number of positions encoded as VInt format. In particular,- * PosVIntCount = totalTermFreq - PackedPosBlockNum*PackedBlockSize- * <li>The procedure how PackedPosDeltaBlock is generated is the same as PackedDocDeltaBlock- * in chapter <a href="#Frequencies">Frequencies and Skip Data</a>.- * <li>PositionDelta is, if payloads are disabled for the term's field, the difference- * between the position of the current occurrence in the document and the previous- * occurrence (or zero, if this is the first occurrence in this document). If payloads- * are enabled for the term's field, then PositionDelta/2 is the difference between the- * current and the previous position. If payloads are enabled and PositionDelta is odd,- * then PayloadLength is stored, indicating the length of the payload at the current- * term position.- * <li>For example, the TermPositions for a term which occurs as the fourth term in one- * document, and as the fifth and ninth term in a subsequent document, would be the- * following sequence of VInts (payloads disabled):- * <p>4, 5, 4- * <li>PayloadData is metadata associated with the current term position. If PayloadLength- * is stored at the current position, then it indicates the length of this payload. If- * PayloadLength is not stored, then this payload has the same length as the payload at- * the previous position.- * <li>OffsetDelta/2 is the difference between this position's startOffset from the previous- * occurrence (or zero, if this is the first occurrence in this document). If- * OffsetDelta is odd, then the length (endOffset-startOffset) differs from the previous- * occurrence and an OffsetLength follows. Offset data is only written for {@link- * IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}.- * </ul>- * </dl>- *- * <a id="Payloads"></a>- *- * <dl>- * <dd><b>Payloads and Offsets</b>- * <p>The .pay file will store payloads and offsets associated with certain term-document- * positions. Some payloads and offsets will be separated out into .pos file, for performance- * reasons.- * <ul>- * <li>PayFile(.pay): --> Header, <TermPayloads, TermOffsets?>- * <sup>TermCount</sup>, Footer- * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}- * <li>TermPayloads --> <PackedPayLengthBlock, SumPayLength, PayData>- * <sup>PackedPayBlockNum</sup>- * <li>TermOffsets --> <PackedOffsetStartDeltaBlock, PackedOffsetLengthBlock>- * <sup>PackedPayBlockNum</sup>- * <li>PackedPayLengthBlock, PackedOffsetStartDeltaBlock, PackedOffsetLengthBlock -->- * {@link PackedInts PackedInts}- * <li>SumPayLength --> {@link DataOutput#writeVInt VInt}- * <li>PayData --> {@link DataOutput#writeByte byte}<sup>SumPayLength</sup>- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}- * </ul>- * <p>Notes:- * <ul>- * <li>The order of TermPayloads/TermOffsets will be the same as TermPositions, note that- * part of payload/offsets are stored in .pos.- * <li>The procedure how PackedPayLengthBlock and PackedOffsetLengthBlock are generated is- * the same as PackedFreqBlock in chapter <a href="#Frequencies">Frequencies and Skip- * Data</a>. While PackedStartDeltaBlock follows a same procedure as- * PackedDocDeltaBlock.- * <li>PackedPayBlockNum is always equal to PackedPosBlockNum, for the same term. It is also- * synonym for PackedOffsetBlockNum.- * <li>SumPayLength is the total length of payloads written within one block, should be the- * sum of PayLengths in one packed block.- * <li>PayLength in PackedPayLengthBlock is the length of each payload associated with the- * current position.- * </ul>- * </dl>- *- * @lucene.experimental- */-public class Lucene50PostingsFormat extends PostingsFormat {-- /**- * Filename extension for document number, frequencies, and skip data. See chapter: <a- * href="#Frequencies">Frequencies and Skip Data</a>- */- public static final String DOC_EXTENSION = "doc";-- /** Filename extension for positions. See chapter: <a href="#Positions">Positions</a> */- public static final String POS_EXTENSION = "pos";-- /**- * Filename extension for payloads and offsets. See chapter: <a href="#Payloads">Payloads and- * Offsets</a>- */- public static final String PAY_EXTENSION = "pay";-- /**- * Expert: The maximum number of skip levels. Smaller values result in slightly smaller indexes,- * but slower skipping in big posting lists.- */- static final int MAX_SKIP_LEVELS = 10;-- static final String TERMS_CODEC = "Lucene50PostingsWriterTerms";- static final String DOC_CODEC = "Lucene50PostingsWriterDoc";- static final String POS_CODEC = "Lucene50PostingsWriterPos";- static final String PAY_CODEC = "Lucene50PostingsWriterPay";-- // Increment version to change it- static final int VERSION_START = 0;- static final int VERSION_IMPACT_SKIP_DATA = 1;- static final int VERSION_CURRENT = VERSION_IMPACT_SKIP_DATA;-- /** Fixed packed block size, number of integers encoded in a single packed block. */- // NOTE: must be multiple of 64 because of PackedInts long-aligned encoding/decoding- public static final int BLOCK_SIZE = 128;-- /** Creates {@code Lucene50PostingsFormat} with default settings. */- public Lucene50PostingsFormat() {- super("Lucene50");- }-- @Override- public String toString() {- return getName() + "(blocksize=" + BLOCK_SIZE + ")";- }-- @Override- public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {- throw new UnsupportedOperationException("Old formats can't be used for writing");- }-- @Override- public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {- PostingsReaderBase postingsReader = new Lucene50PostingsReader(state);- boolean success = false;- try {- FieldsProducer ret = new Lucene40BlockTreeTermsReader(postingsReader, state);- success = true;- return ret;- } finally {- if (!success) {- IOUtils.closeWhileHandlingException(postingsReader);- }- }- }-- /**- * Holds all state required for {@link Lucene50PostingsReader} to produce a {@link- * org.apache.lucene.index.PostingsEnum} without re-seeking the terms dict.- *- * @lucene.internal- */- public static final class IntBlockTermState extends BlockTermState {- /** file pointer to the start of the doc ids enumeration, in {@link #DOC_EXTENSION} file */- public long docStartFP;-- /** file pointer to the start of the positions enumeration, in {@link #POS_EXTENSION} file */- public long posStartFP;-- /** file pointer to the start of the payloads enumeration, in {@link #PAY_EXTENSION} file */- public long payStartFP;-- /**- * file offset for the start of the skip list, relative to docStartFP, if there are more than- * {@link #BLOCK_SIZE} docs; otherwise -1- */- public long skipOffset;-- /**- * file offset for the last position in the last block, if there are more than {@link- * #BLOCK_SIZE} positions; otherwise -1- */- public long lastPosBlockOffset;-- /**- * docid when there is a single pulsed posting, otherwise -1. freq is always implicitly- * totalTermFreq in this case.- */- public int singletonDocID;-- /** Sole constructor. */- public IntBlockTermState() {- skipOffset = -1;- lastPosBlockOffset = -1;- singletonDocID = -1;- }-- @Override- public IntBlockTermState clone() {- IntBlockTermState other = new IntBlockTermState();- other.copyFrom(this);- return other;- }-- @Override- public void copyFrom(TermState _other) {- super.copyFrom(_other);- IntBlockTermState other = (IntBlockTermState) _other;- docStartFP = other.docStartFP;- posStartFP = other.posStartFP;- payStartFP = other.payStartFP;- lastPosBlockOffset = other.lastPosBlockOffset;- skipOffset = other.skipOffset;- singletonDocID = other.singletonDocID;- }-- @Override- public String toString() {- return super.toString()- + " docStartFP="- + docStartFP- + " posStartFP="- + posStartFP- + " payStartFP="- + payStartFP- + " lastPosBlockOffset="- + lastPosBlockOffset- + " singletonDocID="- + singletonDocID;- }- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50PostingsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50PostingsReader.javadeleted file mode 100644index fcf700b4a16..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50PostingsReader.java+++ /dev/null@@ -1,1866 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import static org.apache.lucene.backward_codecs.lucene50.ForUtil.MAX_DATA_SIZE;-import static org.apache.lucene.backward_codecs.lucene50.ForUtil.MAX_ENCODED_SIZE;-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.BLOCK_SIZE;-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.DOC_CODEC;-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.MAX_SKIP_LEVELS;-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.PAY_CODEC;-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.POS_CODEC;-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.TERMS_CODEC;-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.VERSION_CURRENT;-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.VERSION_START;--import java.io.IOException;-import java.util.Arrays;-import org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.IntBlockTermState;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.BlockTermState;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.PostingsReaderBase;-import org.apache.lucene.index.FieldInfo;-import org.apache.lucene.index.Impacts;-import org.apache.lucene.index.ImpactsEnum;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.IndexOptions;-import org.apache.lucene.index.PostingsEnum;-import org.apache.lucene.index.SegmentReadState;-import org.apache.lucene.index.SlowImpactsEnum;-import org.apache.lucene.store.DataInput;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.IOUtils;--/**- * Concrete class that reads docId(maybe frq,pos,offset,payloads) list with postings format.- *- * @lucene.experimental- */-public final class Lucene50PostingsReader extends PostingsReaderBase {-- private final IndexInput docIn;- private final IndexInput posIn;- private final IndexInput payIn;-- final ForUtil forUtil;- private int version;-- /** Sole constructor. */- public Lucene50PostingsReader(SegmentReadState state) throws IOException {- boolean success = false;- IndexInput docIn = null;- IndexInput posIn = null;- IndexInput payIn = null;-- // NOTE: these data files are too costly to verify checksum against all the bytes on open,- // but for now we at least verify proper structure of the checksum footer: which looks- // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption- // such as file truncation.-- String docName =- IndexFileNames.segmentFileName(- state.segmentInfo.name, state.segmentSuffix, Lucene50PostingsFormat.DOC_EXTENSION);- try {- docIn = EndiannessReverserUtil.openInput(state.directory, docName, state.context);- version =- CodecUtil.checkIndexHeader(- docIn,- DOC_CODEC,- VERSION_START,- VERSION_CURRENT,- state.segmentInfo.getId(),- state.segmentSuffix);- forUtil = new ForUtil(docIn);- CodecUtil.retrieveChecksum(docIn);-- if (state.fieldInfos.hasProx()) {- String proxName =- IndexFileNames.segmentFileName(- state.segmentInfo.name, state.segmentSuffix, Lucene50PostingsFormat.POS_EXTENSION);- posIn = EndiannessReverserUtil.openInput(state.directory, proxName, state.context);- CodecUtil.checkIndexHeader(- posIn, POS_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);- CodecUtil.retrieveChecksum(posIn);-- if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {- String payName =- IndexFileNames.segmentFileName(- state.segmentInfo.name,- state.segmentSuffix,- Lucene50PostingsFormat.PAY_EXTENSION);- payIn = EndiannessReverserUtil.openInput(state.directory, payName, state.context);- CodecUtil.checkIndexHeader(- payIn, PAY_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);- CodecUtil.retrieveChecksum(payIn);- }- }-- this.docIn = docIn;- this.posIn = posIn;- this.payIn = payIn;- success = true;- } finally {- if (!success) {- IOUtils.closeWhileHandlingException(docIn, posIn, payIn);- }- }- }-- @Override- public void init(IndexInput termsIn, SegmentReadState state) throws IOException {- // Make sure we are talking to the matching postings writer- CodecUtil.checkIndexHeader(- termsIn,- TERMS_CODEC,- VERSION_START,- VERSION_CURRENT,- state.segmentInfo.getId(),- state.segmentSuffix);- final int indexBlockSize = termsIn.readVInt();- if (indexBlockSize != BLOCK_SIZE) {- throw new IllegalStateException(- "index-time BLOCK_SIZE ("- + indexBlockSize- + ") != read-time BLOCK_SIZE ("- + BLOCK_SIZE- + ")");- }- }-- /** Read values that have been written using variable-length encoding instead of bit-packing. */- static void readVIntBlock(- IndexInput docIn, int[] docBuffer, int[] freqBuffer, int num, boolean indexHasFreq)- throws IOException {- if (indexHasFreq) {- for (int i = 0; i < num; i++) {- final int code = docIn.readVInt();- docBuffer[i] = code >>> 1;- if ((code & 1) != 0) {- freqBuffer[i] = 1;- } else {- freqBuffer[i] = docIn.readVInt();- }- }- } else {- for (int i = 0; i < num; i++) {- docBuffer[i] = docIn.readVInt();- }- }- }-- @Override- public BlockTermState newTermState() {- return new IntBlockTermState();- }-- @Override- public void close() throws IOException {- IOUtils.close(docIn, posIn, payIn);- }-- @Override- public void decodeTerm(- DataInput in, FieldInfo fieldInfo, BlockTermState _termState, boolean absolute)- throws IOException {- final IntBlockTermState termState = (IntBlockTermState) _termState;- final boolean fieldHasPositions =- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;- final boolean fieldHasOffsets =- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)- >= 0;- final boolean fieldHasPayloads = fieldInfo.hasPayloads();-- if (absolute) {- termState.docStartFP = 0;- termState.posStartFP = 0;- termState.payStartFP = 0;- }-- termState.docStartFP += in.readVLong();- if (fieldHasPositions) {- termState.posStartFP += in.readVLong();- if (fieldHasOffsets || fieldHasPayloads) {- termState.payStartFP += in.readVLong();- }- }- if (termState.docFreq == 1) {- termState.singletonDocID = in.readVInt();- } else {- termState.singletonDocID = -1;- }- if (fieldHasPositions) {- if (termState.totalTermFreq > BLOCK_SIZE) {- termState.lastPosBlockOffset = in.readVLong();- } else {- termState.lastPosBlockOffset = -1;- }- }- if (termState.docFreq > BLOCK_SIZE) {- termState.skipOffset = in.readVLong();- } else {- termState.skipOffset = -1;- }- }-- @Override- public PostingsEnum postings(- FieldInfo fieldInfo, BlockTermState termState, PostingsEnum reuse, int flags)- throws IOException {-- boolean indexHasPositions =- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;-- if (indexHasPositions == false- || PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS) == false) {- BlockDocsEnum docsEnum;- if (reuse instanceof BlockDocsEnum) {- docsEnum = (BlockDocsEnum) reuse;- if (!docsEnum.canReuse(docIn, fieldInfo)) {- docsEnum = new BlockDocsEnum(fieldInfo);- }- } else {- docsEnum = new BlockDocsEnum(fieldInfo);- }- return docsEnum.reset((IntBlockTermState) termState, flags);- } else {- EverythingEnum everythingEnum;- if (reuse instanceof EverythingEnum) {- everythingEnum = (EverythingEnum) reuse;- if (!everythingEnum.canReuse(docIn, fieldInfo)) {- everythingEnum = new EverythingEnum(fieldInfo);- }- } else {- everythingEnum = new EverythingEnum(fieldInfo);- }- return everythingEnum.reset((IntBlockTermState) termState, flags);- }- }-- @Override- public ImpactsEnum impacts(FieldInfo fieldInfo, BlockTermState state, int flags)- throws IOException {- if (state.docFreq <= BLOCK_SIZE || version < Lucene50PostingsFormat.VERSION_IMPACT_SKIP_DATA) {- // no skip data- return new SlowImpactsEnum(postings(fieldInfo, state, null, flags));- }-- final boolean indexHasPositions =- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;- final boolean indexHasOffsets =- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)- >= 0;- final boolean indexHasPayloads = fieldInfo.hasPayloads();-- if (indexHasPositions- && PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)- && (indexHasOffsets == false- || PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS) == false)- && (indexHasPayloads == false- || PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS) == false)) {- return new BlockImpactsPostingsEnum(fieldInfo, (IntBlockTermState) state);- }-- return new BlockImpactsEverythingEnum(fieldInfo, (IntBlockTermState) state, flags);- }-- final class BlockDocsEnum extends PostingsEnum {- private final byte[] encoded;-- private final int[] docDeltaBuffer = new int[MAX_DATA_SIZE];- private final int[] freqBuffer = new int[MAX_DATA_SIZE];-- private int docBufferUpto;-- private Lucene50SkipReader skipper;- private boolean skipped;-- final IndexInput startDocIn;-- IndexInput docIn;- final boolean indexHasFreq;- final boolean indexHasPos;- final boolean indexHasOffsets;- final boolean indexHasPayloads;-- private int docFreq; // number of docs in this posting list- private long totalTermFreq; // sum of freqs in this posting list (or docFreq when omitted)- private int docUpto; // how many docs we've read- private int doc; // doc we last read- private int accum; // accumulator for doc deltas-- // Where this term's postings start in the .doc file:- private long docTermStartFP;-- // Where this term's skip data starts (after- // docTermStartFP) in the .doc file (or -1 if there is- // no skip data for this term):- private long skipOffset;-- // docID for next skip point, we won't use skipper if- // target docID is not larger than this- private int nextSkipDoc;-- private boolean needsFreq; // true if the caller actually needs frequencies- // as we read freqs lazily, isFreqsRead shows if freqs are read for the current block- // always true when we don't have freqs (indexHasFreq=false) or don't need freqs- // (needsFreq=false)- private boolean isFreqsRead;- private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1-- public BlockDocsEnum(FieldInfo fieldInfo) throws IOException {- this.startDocIn = Lucene50PostingsReader.this.docIn;- this.docIn = null;- indexHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;- indexHasPos =- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;- indexHasOffsets =- fieldInfo- .getIndexOptions()- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)- >= 0;- indexHasPayloads = fieldInfo.hasPayloads();- encoded = new byte[MAX_ENCODED_SIZE];- }-- public boolean canReuse(IndexInput docIn, FieldInfo fieldInfo) {- return docIn == startDocIn- && indexHasFreq- == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0)- && indexHasPos- == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)- >= 0)- && indexHasPayloads == fieldInfo.hasPayloads();- }-- public PostingsEnum reset(IntBlockTermState termState, int flags) throws IOException {- docFreq = termState.docFreq;- totalTermFreq = indexHasFreq ? termState.totalTermFreq : docFreq;- docTermStartFP = termState.docStartFP;- skipOffset = termState.skipOffset;- singletonDocID = termState.singletonDocID;- if (docFreq > 1) {- if (docIn == null) {- // lazy init- docIn = startDocIn.clone();- }- docIn.seek(docTermStartFP);- }-- doc = -1;- this.needsFreq = PostingsEnum.featureRequested(flags, PostingsEnum.FREQS);- this.isFreqsRead = true;- if (indexHasFreq == false || needsFreq == false) {- Arrays.fill(freqBuffer, 1);- }- accum = 0;- docUpto = 0;- nextSkipDoc = BLOCK_SIZE - 1; // we won't skip if target is found in first block- docBufferUpto = BLOCK_SIZE;- skipped = false;- return this;- }-- @Override- public int freq() throws IOException {- if (isFreqsRead == false) {- forUtil.readBlock(docIn, encoded, freqBuffer); // read freqs for this block- isFreqsRead = true;- }- return freqBuffer[docBufferUpto - 1];- }-- @Override- public int nextPosition() throws IOException {- return -1;- }-- @Override- public int startOffset() throws IOException {- return -1;- }-- @Override- public int endOffset() throws IOException {- return -1;- }-- @Override- public BytesRef getPayload() throws IOException {- return null;- }-- @Override- public int docID() {- return doc;- }-- private void refillDocs() throws IOException {- // Check if we skipped reading the previous block of freqs, and if yes, position docIn after- // it- if (isFreqsRead == false) {- forUtil.skipBlock(docIn);- isFreqsRead = true;- }-- final int left = docFreq - docUpto;- assert left > 0;-- if (left >= BLOCK_SIZE) {- forUtil.readBlock(docIn, encoded, docDeltaBuffer);-- if (indexHasFreq) {- if (needsFreq) {- isFreqsRead = false;- } else {- forUtil.skipBlock(docIn); // skip over freqs if we don't need them at all- }- }- } else if (docFreq == 1) {- docDeltaBuffer[0] = singletonDocID;- freqBuffer[0] = (int) totalTermFreq;- } else {- // Read vInts:- readVIntBlock(docIn, docDeltaBuffer, freqBuffer, left, indexHasFreq);- }- docBufferUpto = 0;- }-- @Override- public int nextDoc() throws IOException {- if (docUpto == docFreq) {- return doc = NO_MORE_DOCS;- }- if (docBufferUpto == BLOCK_SIZE) {- refillDocs(); // we don't need to load freqs for now (will be loaded later if necessary)- }-- accum += docDeltaBuffer[docBufferUpto];- docUpto++;-- doc = accum;- docBufferUpto++;- return doc;- }-- @Override- public int advance(int target) throws IOException {- // current skip docID < docIDs generated from current buffer <= next skip docID- // we don't need to skip if target is buffered already- if (docFreq > BLOCK_SIZE && target > nextSkipDoc) {-- if (skipper == null) {- // Lazy init: first time this enum has ever been used for skipping- skipper =- new Lucene50SkipReader(- version,- docIn.clone(),- MAX_SKIP_LEVELS,- indexHasPos,- indexHasOffsets,- indexHasPayloads);- }-- if (!skipped) {- assert skipOffset != -1;- // This is the first time this enum has skipped- // since reset() was called; load the skip data:- skipper.init(docTermStartFP + skipOffset, docTermStartFP, 0, 0, docFreq);- skipped = true;- }-- // always plus one to fix the result, since skip position in Lucene50SkipReader- // is a little different from MultiLevelSkipListReader- final int newDocUpto = skipper.skipTo(target) + 1;-- if (newDocUpto > docUpto) {- // Skipper moved- assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;- docUpto = newDocUpto;-- // Force to read next block- docBufferUpto = BLOCK_SIZE;- accum = skipper.getDoc(); // actually, this is just lastSkipEntry- docIn.seek(skipper.getDocPointer()); // now point to the block we want to search- // even if freqs were not read from the previous block, we will mark them as read,- // as we don't need to skip the previous block freqs in refillDocs,- // as we have already positioned docIn where in needs to be.- isFreqsRead = true;- }- // next time we call advance, this is used to- // foresee whether skipper is necessary.- nextSkipDoc = skipper.getNextSkipDoc();- }- if (docUpto == docFreq) {- return doc = NO_MORE_DOCS;- }- if (docBufferUpto == BLOCK_SIZE) {- refillDocs();- }-- // Now scan... this is an inlined/pared down version- // of nextDoc():- while (true) {- accum += docDeltaBuffer[docBufferUpto];- docUpto++;-- if (accum >= target) {- break;- }- docBufferUpto++;- if (docUpto == docFreq) {- return doc = NO_MORE_DOCS;- }- }-- docBufferUpto++;- return doc = accum;- }-- @Override- public long cost() {- return docFreq;- }- }-- // Also handles payloads + offsets- final class EverythingEnum extends PostingsEnum {-- private final byte[] encoded;-- private final int[] docDeltaBuffer = new int[MAX_DATA_SIZE];- private final int[] freqBuffer = new int[MAX_DATA_SIZE];- private final int[] posDeltaBuffer = new int[MAX_DATA_SIZE];-- private final int[] payloadLengthBuffer;- private final int[] offsetStartDeltaBuffer;- private final int[] offsetLengthBuffer;-- private byte[] payloadBytes;- private int payloadByteUpto;- private int payloadLength;-- private int lastStartOffset;- private int startOffset;- private int endOffset;-- private int docBufferUpto;- private int posBufferUpto;-- private Lucene50SkipReader skipper;- private boolean skipped;-- final IndexInput startDocIn;-- IndexInput docIn;- final IndexInput posIn;- final IndexInput payIn;- final BytesRef payload;-- final boolean indexHasOffsets;- final boolean indexHasPayloads;-- private int docFreq; // number of docs in this posting list- private long totalTermFreq; // number of positions in this posting list- private int docUpto; // how many docs we've read- private int doc; // doc we last read- private int accum; // accumulator for doc deltas- private int freq; // freq we last read- private int position; // current position-- // how many positions "behind" we are; nextPosition must- // skip these to "catch up":- private int posPendingCount;-- // Lazy pos seek: if != -1 then we must seek to this FP- // before reading positions:- private long posPendingFP;-- // Lazy pay seek: if != -1 then we must seek to this FP- // before reading payloads/offsets:- private long payPendingFP;-- // Where this term's postings start in the .doc file:- private long docTermStartFP;-- // Where this term's postings start in the .pos file:- private long posTermStartFP;-- // Where this term's payloads/offsets start in the .pay- // file:- private long payTermStartFP;-- // File pointer where the last (vInt encoded) pos delta- // block is. We need this to know whether to bulk- // decode vs vInt decode the block:- private long lastPosBlockFP;-- // Where this term's skip data starts (after- // docTermStartFP) in the .doc file (or -1 if there is- // no skip data for this term):- private long skipOffset;-- private int nextSkipDoc;-- private boolean needsOffsets; // true if we actually need offsets- private boolean needsPayloads; // true if we actually need payloads- private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1-- public EverythingEnum(FieldInfo fieldInfo) throws IOException {- indexHasOffsets =- fieldInfo- .getIndexOptions()- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)- >= 0;- indexHasPayloads = fieldInfo.hasPayloads();-- this.startDocIn = Lucene50PostingsReader.this.docIn;- this.docIn = null;- this.posIn = Lucene50PostingsReader.this.posIn.clone();- if (indexHasOffsets || indexHasPayloads) {- this.payIn = Lucene50PostingsReader.this.payIn.clone();- } else {- this.payIn = null;- }- encoded = new byte[MAX_ENCODED_SIZE];- if (indexHasOffsets) {- offsetStartDeltaBuffer = new int[MAX_DATA_SIZE];- offsetLengthBuffer = new int[MAX_DATA_SIZE];- } else {- offsetStartDeltaBuffer = null;- offsetLengthBuffer = null;- startOffset = -1;- endOffset = -1;- }-- if (indexHasPayloads) {- payloadLengthBuffer = new int[MAX_DATA_SIZE];- payloadBytes = new byte[128];- payload = new BytesRef();- } else {- payloadLengthBuffer = null;- payloadBytes = null;- payload = null;- }- }-- public boolean canReuse(IndexInput docIn, FieldInfo fieldInfo) {- return docIn == startDocIn- && indexHasOffsets- == (fieldInfo- .getIndexOptions()- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)- >= 0)- && indexHasPayloads == fieldInfo.hasPayloads();- }-- public EverythingEnum reset(IntBlockTermState termState, int flags) throws IOException {- docFreq = termState.docFreq;- docTermStartFP = termState.docStartFP;- posTermStartFP = termState.posStartFP;- payTermStartFP = termState.payStartFP;- skipOffset = termState.skipOffset;- totalTermFreq = termState.totalTermFreq;- singletonDocID = termState.singletonDocID;- if (docFreq > 1) {- if (docIn == null) {- // lazy init- docIn = startDocIn.clone();- }- docIn.seek(docTermStartFP);- }- posPendingFP = posTermStartFP;- payPendingFP = payTermStartFP;- posPendingCount = 0;- if (termState.totalTermFreq < BLOCK_SIZE) {- lastPosBlockFP = posTermStartFP;- } else if (termState.totalTermFreq == BLOCK_SIZE) {- lastPosBlockFP = -1;- } else {- lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset;- }-- this.needsOffsets = PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS);- this.needsPayloads = PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS);-- doc = -1;- accum = 0;- docUpto = 0;- if (docFreq > BLOCK_SIZE) {- nextSkipDoc = BLOCK_SIZE - 1; // we won't skip if target is found in first block- } else {- nextSkipDoc = NO_MORE_DOCS; // not enough docs for skipping- }- docBufferUpto = BLOCK_SIZE;- skipped = false;- return this;- }-- @Override- public int freq() throws IOException {- return freq;- }-- @Override- public int docID() {- return doc;- }-- private void refillDocs() throws IOException {- final int left = docFreq - docUpto;- assert left > 0;-- if (left >= BLOCK_SIZE) {- forUtil.readBlock(docIn, encoded, docDeltaBuffer);- forUtil.readBlock(docIn, encoded, freqBuffer);- } else if (docFreq == 1) {- docDeltaBuffer[0] = singletonDocID;- freqBuffer[0] = (int) totalTermFreq;- } else {- readVIntBlock(docIn, docDeltaBuffer, freqBuffer, left, true);- }- docBufferUpto = 0;- }-- private void refillPositions() throws IOException {- if (posIn.getFilePointer() == lastPosBlockFP) {- final int count = (int) (totalTermFreq % BLOCK_SIZE);- int payloadLength = 0;- int offsetLength = 0;- payloadByteUpto = 0;- for (int i = 0; i < count; i++) {- int code = posIn.readVInt();- if (indexHasPayloads) {- if ((code & 1) != 0) {- payloadLength = posIn.readVInt();- }- payloadLengthBuffer[i] = payloadLength;- posDeltaBuffer[i] = code >>> 1;- if (payloadLength != 0) {- if (payloadByteUpto + payloadLength > payloadBytes.length) {- payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payloadLength);- }- posIn.readBytes(payloadBytes, payloadByteUpto, payloadLength);- payloadByteUpto += payloadLength;- }- } else {- posDeltaBuffer[i] = code;- }-- if (indexHasOffsets) {- int deltaCode = posIn.readVInt();- if ((deltaCode & 1) != 0) {- offsetLength = posIn.readVInt();- }- offsetStartDeltaBuffer[i] = deltaCode >>> 1;- offsetLengthBuffer[i] = offsetLength;- }- }- payloadByteUpto = 0;- } else {- forUtil.readBlock(posIn, encoded, posDeltaBuffer);-- if (indexHasPayloads) {- if (needsPayloads) {- forUtil.readBlock(payIn, encoded, payloadLengthBuffer);- int numBytes = payIn.readVInt();-- if (numBytes > payloadBytes.length) {- payloadBytes = ArrayUtil.grow(payloadBytes, numBytes);- }- payIn.readBytes(payloadBytes, 0, numBytes);- } else {- // this works, because when writing a vint block we always force the first length to be- // written- forUtil.skipBlock(payIn); // skip over lengths- int numBytes = payIn.readVInt(); // read length of payloadBytes- payIn.seek(payIn.getFilePointer() + numBytes); // skip over payloadBytes- }- payloadByteUpto = 0;- }-- if (indexHasOffsets) {- if (needsOffsets) {- forUtil.readBlock(payIn, encoded, offsetStartDeltaBuffer);- forUtil.readBlock(payIn, encoded, offsetLengthBuffer);- } else {- // this works, because when writing a vint block we always force the first length to be- // written- forUtil.skipBlock(payIn); // skip over starts- forUtil.skipBlock(payIn); // skip over lengths- }- }- }- }-- @Override- public int nextDoc() throws IOException {- if (docUpto == docFreq) {- return doc = NO_MORE_DOCS;- }- if (docBufferUpto == BLOCK_SIZE) {- refillDocs();- }-- accum += docDeltaBuffer[docBufferUpto];- freq = freqBuffer[docBufferUpto];- posPendingCount += freq;- docBufferUpto++;- docUpto++;-- doc = accum;- position = 0;- lastStartOffset = 0;- return doc;- }-- @Override- public int advance(int target) throws IOException {- // TODO: make frq block load lazy/skippable-- if (target > nextSkipDoc) {- if (skipper == null) {- // Lazy init: first time this enum has ever been used for skipping- skipper =- new Lucene50SkipReader(- version, docIn.clone(), MAX_SKIP_LEVELS, true, indexHasOffsets, indexHasPayloads);- }-- if (!skipped) {- assert skipOffset != -1;- // This is the first time this enum has skipped- // since reset() was called; load the skip data:- skipper.init(- docTermStartFP + skipOffset, docTermStartFP, posTermStartFP, payTermStartFP, docFreq);- skipped = true;- }-- final int newDocUpto = skipper.skipTo(target) + 1;-- if (newDocUpto > docUpto) {- // Skipper moved- assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;- docUpto = newDocUpto;-- // Force to read next block- docBufferUpto = BLOCK_SIZE;- accum = skipper.getDoc();- docIn.seek(skipper.getDocPointer());- posPendingFP = skipper.getPosPointer();- payPendingFP = skipper.getPayPointer();- posPendingCount = skipper.getPosBufferUpto();- lastStartOffset = 0; // new document- payloadByteUpto = skipper.getPayloadByteUpto();- }- nextSkipDoc = skipper.getNextSkipDoc();- }- if (docUpto == docFreq) {- return doc = NO_MORE_DOCS;- }- if (docBufferUpto == BLOCK_SIZE) {- refillDocs();- }-- // Now scan:- while (true) {- accum += docDeltaBuffer[docBufferUpto];- freq = freqBuffer[docBufferUpto];- posPendingCount += freq;- docBufferUpto++;- docUpto++;-- if (accum >= target) {- break;- }- if (docUpto == docFreq) {- return doc = NO_MORE_DOCS;- }- }-- position = 0;- lastStartOffset = 0;- return doc = accum;- }-- // TODO: in theory we could avoid loading frq block- // when not needed, ie, use skip data to load how far to- // seek the pos pointer ... instead of having to load frq- // blocks only to sum up how many positions to skip- private void skipPositions() throws IOException {- // Skip positions now:- int toSkip = posPendingCount - freq;- // if (DEBUG) {- // System.out.println(" FPR.skipPositions: toSkip=" + toSkip);- // }-- final int leftInBlock = BLOCK_SIZE - posBufferUpto;- if (toSkip < leftInBlock) {- int end = posBufferUpto + toSkip;- while (posBufferUpto < end) {- if (indexHasPayloads) {- payloadByteUpto += payloadLengthBuffer[posBufferUpto];- }- posBufferUpto++;- }- } else {- toSkip -= leftInBlock;- while (toSkip >= BLOCK_SIZE) {- assert posIn.getFilePointer() != lastPosBlockFP;- forUtil.skipBlock(posIn);-- if (indexHasPayloads) {- // Skip payloadLength block:- forUtil.skipBlock(payIn);-- // Skip payloadBytes block:- int numBytes = payIn.readVInt();- payIn.seek(payIn.getFilePointer() + numBytes);- }-- if (indexHasOffsets) {- forUtil.skipBlock(payIn);- forUtil.skipBlock(payIn);- }- toSkip -= BLOCK_SIZE;- }- refillPositions();- payloadByteUpto = 0;- posBufferUpto = 0;- while (posBufferUpto < toSkip) {- if (indexHasPayloads) {- payloadByteUpto += payloadLengthBuffer[posBufferUpto];- }- posBufferUpto++;- }- }-- position = 0;- lastStartOffset = 0;- }-- @Override- public int nextPosition() throws IOException {- assert posPendingCount > 0;-- if (posPendingFP != -1) {- posIn.seek(posPendingFP);- posPendingFP = -1;-- if (payPendingFP != -1 && payIn != null) {- payIn.seek(payPendingFP);- payPendingFP = -1;- }-- // Force buffer refill:- posBufferUpto = BLOCK_SIZE;- }-- if (posPendingCount > freq) {- skipPositions();- posPendingCount = freq;- }-- if (posBufferUpto == BLOCK_SIZE) {- refillPositions();- posBufferUpto = 0;- }- position += posDeltaBuffer[posBufferUpto];-- if (indexHasPayloads) {- payloadLength = payloadLengthBuffer[posBufferUpto];- payload.bytes = payloadBytes;- payload.offset = payloadByteUpto;- payload.length = payloadLength;- payloadByteUpto += payloadLength;- }-- if (indexHasOffsets) {- startOffset = lastStartOffset + offsetStartDeltaBuffer[posBufferUpto];- endOffset = startOffset + offsetLengthBuffer[posBufferUpto];- lastStartOffset = startOffset;- }-- posBufferUpto++;- posPendingCount--;- return position;- }-- @Override- public int startOffset() {- return startOffset;- }-- @Override- public int endOffset() {- return endOffset;- }-- @Override- public BytesRef getPayload() {- if (payloadLength == 0) {- return null;- } else {- return payload;- }- }-- @Override- public long cost() {- return docFreq;- }- }-- final class BlockImpactsPostingsEnum extends ImpactsEnum {-- private final byte[] encoded;-- private final int[] docDeltaBuffer = new int[MAX_DATA_SIZE];- private final int[] freqBuffer = new int[MAX_DATA_SIZE];- private final int[] posDeltaBuffer = new int[MAX_DATA_SIZE];-- private int docBufferUpto;- private int posBufferUpto;-- private final Lucene50ScoreSkipReader skipper;-- final IndexInput docIn;- final IndexInput posIn;-- final boolean indexHasOffsets;- final boolean indexHasPayloads;-- private int docFreq; // number of docs in this posting list- private long totalTermFreq; // number of positions in this posting list- private int docUpto; // how many docs we've read- private int doc; // doc we last read- private int accum; // accumulator for doc deltas- private int freq; // freq we last read- private int position; // current position-- // how many positions "behind" we are; nextPosition must- // skip these to "catch up":- private int posPendingCount;-- // Lazy pos seek: if != -1 then we must seek to this FP- // before reading positions:- private long posPendingFP;-- // Where this term's postings start in the .doc file:- private long docTermStartFP;-- // Where this term's postings start in the .pos file:- private long posTermStartFP;-- // Where this term's payloads/offsets start in the .pay- // file:- private long payTermStartFP;-- // File pointer where the last (vInt encoded) pos delta- // block is. We need this to know whether to bulk- // decode vs vInt decode the block:- private long lastPosBlockFP;-- private int nextSkipDoc = -1;-- private long seekTo = -1;-- public BlockImpactsPostingsEnum(FieldInfo fieldInfo, IntBlockTermState termState)- throws IOException {- indexHasOffsets =- fieldInfo- .getIndexOptions()- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)- >= 0;- indexHasPayloads = fieldInfo.hasPayloads();-- this.docIn = Lucene50PostingsReader.this.docIn.clone();-- encoded = new byte[MAX_ENCODED_SIZE];-- this.posIn = Lucene50PostingsReader.this.posIn.clone();-- docFreq = termState.docFreq;- docTermStartFP = termState.docStartFP;- posTermStartFP = termState.posStartFP;- payTermStartFP = termState.payStartFP;- totalTermFreq = termState.totalTermFreq;- docIn.seek(docTermStartFP);- posPendingFP = posTermStartFP;- posPendingCount = 0;- if (termState.totalTermFreq < BLOCK_SIZE) {- lastPosBlockFP = posTermStartFP;- } else if (termState.totalTermFreq == BLOCK_SIZE) {- lastPosBlockFP = -1;- } else {- lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset;- }-- doc = -1;- accum = 0;- docUpto = 0;- docBufferUpto = BLOCK_SIZE;-- skipper =- new Lucene50ScoreSkipReader(- version, docIn.clone(), MAX_SKIP_LEVELS, true, indexHasOffsets, indexHasPayloads);- skipper.init(- docTermStartFP + termState.skipOffset,- docTermStartFP,- posTermStartFP,- payTermStartFP,- docFreq);- }-- @Override- public int freq() throws IOException {- return freq;- }-- @Override- public int docID() {- return doc;- }-- private void refillDocs() throws IOException {- final int left = docFreq - docUpto;- assert left > 0;-- if (left >= BLOCK_SIZE) {- forUtil.readBlock(docIn, encoded, docDeltaBuffer);- forUtil.readBlock(docIn, encoded, freqBuffer);- } else {- readVIntBlock(docIn, docDeltaBuffer, freqBuffer, left, true);- }- docBufferUpto = 0;- }-- private void refillPositions() throws IOException {- if (posIn.getFilePointer() == lastPosBlockFP) {- final int count = (int) (totalTermFreq % BLOCK_SIZE);- int payloadLength = 0;- for (int i = 0; i < count; i++) {- int code = posIn.readVInt();- if (indexHasPayloads) {- if ((code & 1) != 0) {- payloadLength = posIn.readVInt();- }- posDeltaBuffer[i] = code >>> 1;- if (payloadLength != 0) {- posIn.seek(posIn.getFilePointer() + payloadLength);- }- } else {- posDeltaBuffer[i] = code;- }- if (indexHasOffsets) {- if ((posIn.readVInt() & 1) != 0) {- // offset length changed- posIn.readVInt();- }- }- }- } else {- forUtil.readBlock(posIn, encoded, posDeltaBuffer);- }- }-- @Override- public void advanceShallow(int target) throws IOException {- if (target > nextSkipDoc) {- // always plus one to fix the result, since skip position in Lucene50SkipReader- // is a little different from MultiLevelSkipListReader- final int newDocUpto = skipper.skipTo(target) + 1;-- if (newDocUpto > docUpto) {- // Skipper moved- assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;- docUpto = newDocUpto;-- // Force to read next block- docBufferUpto = BLOCK_SIZE;- accum = skipper.getDoc();- posPendingFP = skipper.getPosPointer();- posPendingCount = skipper.getPosBufferUpto();- seekTo = skipper.getDocPointer(); // delay the seek- }- // next time we call advance, this is used to- // foresee whether skipper is necessary.- nextSkipDoc = skipper.getNextSkipDoc();- }- assert nextSkipDoc >= target;- }-- @Override- public Impacts getImpacts() throws IOException {- advanceShallow(doc);- return skipper.getImpacts();- }-- @Override- public int nextDoc() throws IOException {- return advance(doc + 1);- }-- @Override- public int advance(int target) throws IOException {- if (target > nextSkipDoc) {- advanceShallow(target);- }- if (docUpto == docFreq) {- return doc = NO_MORE_DOCS;- }- if (docBufferUpto == BLOCK_SIZE) {- if (seekTo >= 0) {- docIn.seek(seekTo);- seekTo = -1;- }- refillDocs();- }-- // Now scan:- while (true) {- accum += docDeltaBuffer[docBufferUpto];- freq = freqBuffer[docBufferUpto];- posPendingCount += freq;- docBufferUpto++;- docUpto++;-- if (accum >= target) {- break;- }- if (docUpto == docFreq) {- return doc = NO_MORE_DOCS;- }- }- position = 0;-- return doc = accum;- }-- // TODO: in theory we could avoid loading frq block- // when not needed, ie, use skip data to load how far to- // seek the pos pointer ... instead of having to load frq- // blocks only to sum up how many positions to skip- private void skipPositions() throws IOException {- // Skip positions now:- int toSkip = posPendingCount - freq;-- final int leftInBlock = BLOCK_SIZE - posBufferUpto;- if (toSkip < leftInBlock) {- posBufferUpto += toSkip;- } else {- toSkip -= leftInBlock;- while (toSkip >= BLOCK_SIZE) {- assert posIn.getFilePointer() != lastPosBlockFP;- forUtil.skipBlock(posIn);- toSkip -= BLOCK_SIZE;- }- refillPositions();- posBufferUpto = toSkip;- }-- position = 0;- }-- @Override- public int nextPosition() throws IOException {- assert posPendingCount > 0;-- if (posPendingFP != -1) {- posIn.seek(posPendingFP);- posPendingFP = -1;-- // Force buffer refill:- posBufferUpto = BLOCK_SIZE;- }-- if (posPendingCount > freq) {- skipPositions();- posPendingCount = freq;- }-- if (posBufferUpto == BLOCK_SIZE) {- refillPositions();- posBufferUpto = 0;- }- position += posDeltaBuffer[posBufferUpto++];-- posPendingCount--;- return position;- }-- @Override- public int startOffset() {- return -1;- }-- @Override- public int endOffset() {- return -1;- }-- @Override- public BytesRef getPayload() {- return null;- }-- @Override- public long cost() {- return docFreq;- }- }-- final class BlockImpactsEverythingEnum extends ImpactsEnum {-- private final byte[] encoded;-- private final int[] docDeltaBuffer = new int[MAX_DATA_SIZE];- private final int[] freqBuffer = new int[MAX_DATA_SIZE];- private final int[] posDeltaBuffer = new int[MAX_DATA_SIZE];-- private final int[] payloadLengthBuffer;- private final int[] offsetStartDeltaBuffer;- private final int[] offsetLengthBuffer;-- private byte[] payloadBytes;- private int payloadByteUpto;- private int payloadLength;-- private int lastStartOffset;- private int startOffset = -1;- private int endOffset = -1;-- private int docBufferUpto;- private int posBufferUpto;-- private final Lucene50ScoreSkipReader skipper;-- final IndexInput docIn;- final IndexInput posIn;- final IndexInput payIn;- final BytesRef payload;-- final boolean indexHasFreq;- final boolean indexHasPos;- final boolean indexHasOffsets;- final boolean indexHasPayloads;-- private int docFreq; // number of docs in this posting list- private long totalTermFreq; // number of positions in this posting list- private int docUpto; // how many docs we've read- private int posDocUpTo; // for how many docs we've read positions, offsets, and payloads- private int doc; // doc we last read- private int accum; // accumulator for doc deltas- private int position; // current position-- // how many positions "behind" we are; nextPosition must- // skip these to "catch up":- private int posPendingCount;-- // Lazy pos seek: if != -1 then we must seek to this FP- // before reading positions:- private long posPendingFP;-- // Lazy pay seek: if != -1 then we must seek to this FP- // before reading payloads/offsets:- private long payPendingFP;-- // Where this term's postings start in the .doc file:- private long docTermStartFP;-- // Where this term's postings start in the .pos file:- private long posTermStartFP;-- // Where this term's payloads/offsets start in the .pay- // file:- private long payTermStartFP;-- // File pointer where the last (vInt encoded) pos delta- // block is. We need this to know whether to bulk- // decode vs vInt decode the block:- private long lastPosBlockFP;-- private int nextSkipDoc = -1;-- private final boolean needsPositions;- private final boolean needsOffsets; // true if we actually need offsets- private final boolean needsPayloads; // true if we actually need payloads-- private boolean- isFreqsRead; // shows if freqs for the current doc block are read into freqBuffer-- private long seekTo = -1;-- public BlockImpactsEverythingEnum(FieldInfo fieldInfo, IntBlockTermState termState, int flags)- throws IOException {- indexHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;- indexHasPos =- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;- indexHasOffsets =- fieldInfo- .getIndexOptions()- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)- >= 0;- indexHasPayloads = fieldInfo.hasPayloads();-- needsPositions = PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS);- needsOffsets = PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS);- needsPayloads = PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS);-- this.docIn = Lucene50PostingsReader.this.docIn.clone();-- encoded = new byte[MAX_ENCODED_SIZE];-- if (indexHasPos && needsPositions) {- this.posIn = Lucene50PostingsReader.this.posIn.clone();- } else {- this.posIn = null;- }-- if ((indexHasOffsets && needsOffsets) || (indexHasPayloads && needsPayloads)) {- this.payIn = Lucene50PostingsReader.this.payIn.clone();- } else {- this.payIn = null;- }-- if (indexHasOffsets) {- offsetStartDeltaBuffer = new int[MAX_DATA_SIZE];- offsetLengthBuffer = new int[MAX_DATA_SIZE];- } else {- offsetStartDeltaBuffer = null;- offsetLengthBuffer = null;- startOffset = -1;- endOffset = -1;- }-- if (indexHasPayloads) {- payloadLengthBuffer = new int[MAX_DATA_SIZE];- payloadBytes = new byte[128];- payload = new BytesRef();- } else {- payloadLengthBuffer = null;- payloadBytes = null;- payload = null;- }-- docFreq = termState.docFreq;- docTermStartFP = termState.docStartFP;- posTermStartFP = termState.posStartFP;- payTermStartFP = termState.payStartFP;- totalTermFreq = termState.totalTermFreq;- docIn.seek(docTermStartFP);- posPendingFP = posTermStartFP;- payPendingFP = payTermStartFP;- posPendingCount = 0;- if (termState.totalTermFreq < BLOCK_SIZE) {- lastPosBlockFP = posTermStartFP;- } else if (termState.totalTermFreq == BLOCK_SIZE) {- lastPosBlockFP = -1;- } else {- lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset;- }-- doc = -1;- accum = 0;- docUpto = 0;- posDocUpTo = 0;- isFreqsRead = true;- docBufferUpto = BLOCK_SIZE;-- skipper =- new Lucene50ScoreSkipReader(- version,- docIn.clone(),- MAX_SKIP_LEVELS,- indexHasPos,- indexHasOffsets,- indexHasPayloads);- skipper.init(- docTermStartFP + termState.skipOffset,- docTermStartFP,- posTermStartFP,- payTermStartFP,- docFreq);-- if (indexHasFreq == false) {- Arrays.fill(freqBuffer, 1);- }- }-- @Override- public int freq() throws IOException {- if (indexHasFreq && (isFreqsRead == false)) {- forUtil.readBlock(docIn, encoded, freqBuffer); // read freqs for this block- isFreqsRead = true;- }- return freqBuffer[docBufferUpto - 1];- }-- @Override- public int docID() {- return doc;- }-- private void refillDocs() throws IOException {- if (indexHasFreq) {- if (isFreqsRead == false) { // previous freq block was not read- // check if we need to load the previous freq block to catch up on positions or we can- // skip it- if (indexHasPos && needsPositions && (posDocUpTo < docUpto)) {- forUtil.readBlock(docIn, encoded, freqBuffer); // load the previous freq block- } else {- forUtil.skipBlock(docIn); // skip it- }- isFreqsRead = true;- }- if (indexHasPos && needsPositions) {- while (posDocUpTo- < docUpto) { // catch on positions, bring posPendingCount upto the current doc- posPendingCount += freqBuffer[docBufferUpto - (docUpto - posDocUpTo)];- posDocUpTo++;- }- }- }-- final int left = docFreq - docUpto;- assert left > 0;-- if (left >= BLOCK_SIZE) {- forUtil.readBlock(docIn, encoded, docDeltaBuffer);- if (indexHasFreq) {- isFreqsRead =- false; // freq block will be loaded lazily when necessary, we don't load it here- }- } else {- readVIntBlock(docIn, docDeltaBuffer, freqBuffer, left, indexHasFreq);- }- docBufferUpto = 0;- }-- private void refillPositions() throws IOException {- if (posIn.getFilePointer() == lastPosBlockFP) {- final int count = (int) (totalTermFreq % BLOCK_SIZE);- int payloadLength = 0;- int offsetLength = 0;- payloadByteUpto = 0;- for (int i = 0; i < count; i++) {- int code = posIn.readVInt();- if (indexHasPayloads) {- if ((code & 1) != 0) {- payloadLength = posIn.readVInt();- }- payloadLengthBuffer[i] = payloadLength;- posDeltaBuffer[i] = code >>> 1;- if (payloadLength != 0) {- if (payloadByteUpto + payloadLength > payloadBytes.length) {- payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payloadLength);- }- posIn.readBytes(payloadBytes, payloadByteUpto, payloadLength);- payloadByteUpto += payloadLength;- }- } else {- posDeltaBuffer[i] = code;- }-- if (indexHasOffsets) {- int deltaCode = posIn.readVInt();- if ((deltaCode & 1) != 0) {- offsetLength = posIn.readVInt();- }- offsetStartDeltaBuffer[i] = deltaCode >>> 1;- offsetLengthBuffer[i] = offsetLength;- }- }- payloadByteUpto = 0;- } else {- forUtil.readBlock(posIn, encoded, posDeltaBuffer);-- if (indexHasPayloads && payIn != null) {- if (needsPayloads) {- forUtil.readBlock(payIn, encoded, payloadLengthBuffer);- int numBytes = payIn.readVInt();-- if (numBytes > payloadBytes.length) {- payloadBytes = ArrayUtil.grow(payloadBytes, numBytes);- }- payIn.readBytes(payloadBytes, 0, numBytes);- } else {- // this works, because when writing a vint block we always force the first length to be- // written- forUtil.skipBlock(payIn); // skip over lengths- int numBytes = payIn.readVInt(); // read length of payloadBytes- payIn.seek(payIn.getFilePointer() + numBytes); // skip over payloadBytes- }- payloadByteUpto = 0;- }-- if (indexHasOffsets && payIn != null) {- if (needsOffsets) {- forUtil.readBlock(payIn, encoded, offsetStartDeltaBuffer);- forUtil.readBlock(payIn, encoded, offsetLengthBuffer);- } else {- // this works, because when writing a vint block we always force the first length to be- // written- forUtil.skipBlock(payIn); // skip over starts- forUtil.skipBlock(payIn); // skip over lengths- }- }- }- }-- @Override- public void advanceShallow(int target) throws IOException {- if (target > nextSkipDoc) {- // always plus one to fix the result, since skip position in Lucene50SkipReader- // is a little different from MultiLevelSkipListReader- final int newDocUpto = skipper.skipTo(target) + 1;-- if (newDocUpto > docUpto) {- // Skipper moved- assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;- docUpto = newDocUpto;- posDocUpTo = docUpto;-- // Force to read next block- docBufferUpto = BLOCK_SIZE;- accum = skipper.getDoc();- posPendingFP = skipper.getPosPointer();- payPendingFP = skipper.getPayPointer();- posPendingCount = skipper.getPosBufferUpto();- lastStartOffset = 0; // new document- payloadByteUpto = skipper.getPayloadByteUpto(); // actually, this is just lastSkipEntry- seekTo = skipper.getDocPointer(); // delay the seek- }- // next time we call advance, this is used to- // foresee whether skipper is necessary.- nextSkipDoc = skipper.getNextSkipDoc();- }- assert nextSkipDoc >= target;- }-- @Override- public Impacts getImpacts() throws IOException {- advanceShallow(doc);- return skipper.getImpacts();- }-- @Override- public int nextDoc() throws IOException {- return advance(doc + 1);- }-- @Override- public int advance(int target) throws IOException {- if (target > nextSkipDoc) {- advanceShallow(target);- }- if (docUpto == docFreq) {- return doc = NO_MORE_DOCS;- }- if (docBufferUpto == BLOCK_SIZE) {- if (seekTo >= 0) {- docIn.seek(seekTo);- seekTo = -1;- isFreqsRead = true; // reset isFreqsRead- }- refillDocs();- }-- // Now scan:- while (true) {- accum += docDeltaBuffer[docBufferUpto];- docBufferUpto++;- docUpto++;-- if (accum >= target) {- break;- }- if (docUpto == docFreq) {- return doc = NO_MORE_DOCS;- }- }- position = 0;- lastStartOffset = 0;-- return doc = accum;- }-- // TODO: in theory we could avoid loading frq block- // when not needed, ie, use skip data to load how far to- // seek the pos pointer ... instead of having to load frq- // blocks only to sum up how many positions to skip- private void skipPositions() throws IOException {- // Skip positions now:- int toSkip = posPendingCount - freqBuffer[docBufferUpto - 1];- // if (DEBUG) {- // System.out.println(" FPR.skipPositions: toSkip=" + toSkip);- // }-- final int leftInBlock = BLOCK_SIZE - posBufferUpto;- if (toSkip < leftInBlock) {- int end = posBufferUpto + toSkip;- while (posBufferUpto < end) {- if (indexHasPayloads) {- payloadByteUpto += payloadLengthBuffer[posBufferUpto];- }- posBufferUpto++;- }- } else {- toSkip -= leftInBlock;- while (toSkip >= BLOCK_SIZE) {- assert posIn.getFilePointer() != lastPosBlockFP;- forUtil.skipBlock(posIn);-- if (indexHasPayloads && payIn != null) {- // Skip payloadLength block:- forUtil.skipBlock(payIn);-- // Skip payloadBytes block:- int numBytes = payIn.readVInt();- payIn.seek(payIn.getFilePointer() + numBytes);- }-- if (indexHasOffsets && payIn != null) {- forUtil.skipBlock(payIn);- forUtil.skipBlock(payIn);- }- toSkip -= BLOCK_SIZE;- }- refillPositions();- payloadByteUpto = 0;- posBufferUpto = 0;- while (posBufferUpto < toSkip) {- if (indexHasPayloads) {- payloadByteUpto += payloadLengthBuffer[posBufferUpto];- }- posBufferUpto++;- }- }-- position = 0;- lastStartOffset = 0;- }-- @Override- public int nextPosition() throws IOException {- if (indexHasPos == false || needsPositions == false) {- return -1;- }-- if (isFreqsRead == false) {- forUtil.readBlock(docIn, encoded, freqBuffer); // read freqs for this docs block- isFreqsRead = true;- }- while (posDocUpTo < docUpto) { // bring posPendingCount upto the current doc- posPendingCount += freqBuffer[docBufferUpto - (docUpto - posDocUpTo)];- posDocUpTo++;- }-- assert posPendingCount > 0;-- if (posPendingFP != -1) {- posIn.seek(posPendingFP);- posPendingFP = -1;-- if (payPendingFP != -1 && payIn != null) {- payIn.seek(payPendingFP);- payPendingFP = -1;- }-- // Force buffer refill:- posBufferUpto = BLOCK_SIZE;- }-- if (posPendingCount > freqBuffer[docBufferUpto - 1]) {- skipPositions();- posPendingCount = freqBuffer[docBufferUpto - 1];- }-- if (posBufferUpto == BLOCK_SIZE) {- refillPositions();- posBufferUpto = 0;- }- position += posDeltaBuffer[posBufferUpto];-- if (indexHasPayloads) {- payloadLength = payloadLengthBuffer[posBufferUpto];- payload.bytes = payloadBytes;- payload.offset = payloadByteUpto;- payload.length = payloadLength;- payloadByteUpto += payloadLength;- }-- if (indexHasOffsets && needsOffsets) {- startOffset = lastStartOffset + offsetStartDeltaBuffer[posBufferUpto];- endOffset = startOffset + offsetLengthBuffer[posBufferUpto];- lastStartOffset = startOffset;- }-- posBufferUpto++;- posPendingCount--;- return position;- }-- @Override- public int startOffset() {- return startOffset;- }-- @Override- public int endOffset() {- return endOffset;- }-- @Override- public BytesRef getPayload() {- if (payloadLength == 0) {- return null;- } else {- return payload;- }- }-- @Override- public long cost() {- return docFreq;- }- }-- @Override- public void checkIntegrity() throws IOException {- if (docIn != null) {- CodecUtil.checksumEntireFile(docIn);- }- if (posIn != null) {- CodecUtil.checksumEntireFile(posIn);- }- if (payIn != null) {- CodecUtil.checksumEntireFile(payIn);- }- }-- @Override- public String toString() {- return getClass().getSimpleName()- + "(positions="- + (posIn != null)- + ",payloads="- + (payIn != null)- + ")";- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50ScoreSkipReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50ScoreSkipReader.javadeleted file mode 100644index 11bdb595fc5..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50ScoreSkipReader.java+++ /dev/null@@ -1,139 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import java.io.IOException;-import java.util.Arrays;-import org.apache.lucene.index.FreqAndNormBuffer;-import org.apache.lucene.index.Impacts;-import org.apache.lucene.store.ByteArrayDataInput;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.util.ArrayUtil;--final class Lucene50ScoreSkipReader extends Lucene50SkipReader {-- private final byte[][] impactData;- private final int[] impactDataLength;- private final ByteArrayDataInput badi = new ByteArrayDataInput();- private final Impacts impacts;- private int numLevels = 1;- private final FreqAndNormBuffer[] perLevelImpacts;-- public Lucene50ScoreSkipReader(- int version,- IndexInput skipStream,- int maxSkipLevels,- boolean hasPos,- boolean hasOffsets,- boolean hasPayloads) {- super(version, skipStream, maxSkipLevels, hasPos, hasOffsets, hasPayloads);- if (version < Lucene50PostingsFormat.VERSION_IMPACT_SKIP_DATA) {- throw new IllegalStateException("Cannot skip based on scores if impacts are not indexed");- }- this.impactData = new byte[maxSkipLevels][];- Arrays.fill(impactData, new byte[0]);- this.impactDataLength = new int[maxSkipLevels];- this.perLevelImpacts = new FreqAndNormBuffer[maxSkipLevels];- for (int i = 0; i < perLevelImpacts.length; ++i) {- perLevelImpacts[i] = new FreqAndNormBuffer();- perLevelImpacts[i].add(Integer.MAX_VALUE, 1L);- }- impacts =- new Impacts() {-- @Override- public int numLevels() {- return numLevels;- }-- @Override- public int getDocIdUpTo(int level) {- return skipDoc[level];- }-- @Override- public FreqAndNormBuffer getImpacts(int level) {- assert level < numLevels;- if (impactDataLength[level] > 0) {- badi.reset(impactData[level], 0, impactDataLength[level]);- perLevelImpacts[level] = readImpacts(badi, perLevelImpacts[level]);- impactDataLength[level] = 0;- }- return perLevelImpacts[level];- }- };- }-- @Override- public int skipTo(int target) throws IOException {- int result = super.skipTo(target);- if (numberOfSkipLevels > 0) {- numLevels = numberOfSkipLevels;- } else {- // End of postings don't have skip data anymore, so we fill with dummy data- // like SlowImpactsEnum.- numLevels = 1;- perLevelImpacts[0].size = 1;- perLevelImpacts[0].freqs[0] = Integer.MAX_VALUE;- perLevelImpacts[0].norms[0] = 1L;- impactDataLength[0] = 0;- }- return result;- }-- Impacts getImpacts() {- return impacts;- }-- @Override- protected void readImpacts(int level, IndexInput skipStream) throws IOException {- int length = skipStream.readVInt();- if (impactData[level].length < length) {- impactData[level] = new byte[ArrayUtil.oversize(length, Byte.BYTES)];- }- skipStream.readBytes(impactData[level], 0, length);- impactDataLength[level] = length;- }-- static FreqAndNormBuffer readImpacts(ByteArrayDataInput in, FreqAndNormBuffer reuse) {- int maxNumImpacts = in.length(); // at most one impact per byte- reuse.growNoCopy(maxNumImpacts);-- int freq = 0;- long norm = 0;- int size = 0;- while (in.getPosition() < in.length()) {- int freqDelta = in.readVInt();- if ((freqDelta & 0x01) != 0) {- freq += 1 + (freqDelta >>> 1);- try {- norm += 1 + in.readZLong();- } catch (IOException e) {- throw new RuntimeException(e); // cannot happen on a BADI- }- } else {- freq += 1 + (freqDelta >>> 1);- norm++;- }- reuse.freqs[size] = freq;- reuse.norms[size] = norm;- size++;- }- reuse.size = size;- return reuse;- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50SkipReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50SkipReader.javadeleted file mode 100644index a74eacdd21d..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50SkipReader.java+++ /dev/null@@ -1,214 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.BLOCK_SIZE;--import java.io.IOException;-import java.util.Arrays;-import org.apache.lucene.codecs.MultiLevelSkipListReader;-import org.apache.lucene.store.IndexInput;--/**- * Implements the skip list reader for block postings format that stores positions and payloads.- *- * <p>Although this skipper uses MultiLevelSkipListReader as an interface, its definition of skip- * position will be a little different.- *- * <p>For example, when skipInterval = blockSize = 3, df = 2*skipInterval = 6,- *- * <pre>- * 0 1 2 3 4 5- * d d d d d d (posting list)- * ^ ^ (skip point in MultiLeveSkipWriter)- * ^ (skip point in Lucene50SkipWriter)- * </pre>- *- * In this case, MultiLevelSkipListReader will use the last document as a skip point, while- * Lucene50SkipReader should assume no skip point will comes.- *- * <p>If we use the interface directly in Lucene50SkipReader, it may silly try to read another skip- * data after the only skip point is loaded.- *- * <p>To illustrate this, we can call skipTo(d[5]), since skip point d[3] has smaller docId, and- * numSkipped+blockSize== df, the MultiLevelSkipListReader will assume the skip list isn't exhausted- * yet, and try to load a non-existed skip point- *- * <p>Therefore, we'll trim df before passing it to the interface. see trim(int)- */-class Lucene50SkipReader extends MultiLevelSkipListReader {- private final int version;- private long[] docPointer;- private long[] posPointer;- private long[] payPointer;- private int[] posBufferUpto;- private int[] payloadByteUpto;-- private long lastPosPointer;- private long lastPayPointer;- private int lastPayloadByteUpto;- private long lastDocPointer;- private int lastPosBufferUpto;-- public Lucene50SkipReader(- int version,- IndexInput skipStream,- int maxSkipLevels,- boolean hasPos,- boolean hasOffsets,- boolean hasPayloads) {- super(skipStream, maxSkipLevels, BLOCK_SIZE, 8);- this.version = version;- docPointer = new long[maxSkipLevels];- if (hasPos) {- posPointer = new long[maxSkipLevels];- posBufferUpto = new int[maxSkipLevels];- if (hasPayloads) {- payloadByteUpto = new int[maxSkipLevels];- } else {- payloadByteUpto = null;- }- if (hasOffsets || hasPayloads) {- payPointer = new long[maxSkipLevels];- } else {- payPointer = null;- }- } else {- posPointer = null;- }- }-- /**- * Trim original docFreq to tell skipReader read proper number of skip points.- *- * <p>Since our definition in Lucene50Skip* is a little different from MultiLevelSkip* This- * trimmed docFreq will prevent skipReader from: 1. silly reading a non-existed skip point after- * the last block boundary 2. moving into the vInt block- */- protected int trim(int df) {- return df % BLOCK_SIZE == 0 ? df - 1 : df;- }-- public void init(- long skipPointer, long docBasePointer, long posBasePointer, long payBasePointer, int df)- throws IOException {- super.init(skipPointer, trim(df));- lastDocPointer = docBasePointer;- lastPosPointer = posBasePointer;- lastPayPointer = payBasePointer;-- Arrays.fill(docPointer, docBasePointer);- if (posPointer != null) {- Arrays.fill(posPointer, posBasePointer);- if (payPointer != null) {- Arrays.fill(payPointer, payBasePointer);- }- } else {- assert posBasePointer == 0;- }- }-- /**- * Returns the doc pointer of the doc to which the last call of {@link- * MultiLevelSkipListReader#skipTo(int)} has skipped.- */- public long getDocPointer() {- return lastDocPointer;- }-- public long getPosPointer() {- return lastPosPointer;- }-- public int getPosBufferUpto() {- return lastPosBufferUpto;- }-- public long getPayPointer() {- return lastPayPointer;- }-- public int getPayloadByteUpto() {- return lastPayloadByteUpto;- }-- public int getNextSkipDoc() {- return skipDoc[0];- }-- @Override- protected void seekChild(int level) throws IOException {- super.seekChild(level);- docPointer[level] = lastDocPointer;- if (posPointer != null) {- posPointer[level] = lastPosPointer;- posBufferUpto[level] = lastPosBufferUpto;- if (payloadByteUpto != null) {- payloadByteUpto[level] = lastPayloadByteUpto;- }- if (payPointer != null) {- payPointer[level] = lastPayPointer;- }- }- }-- @Override- protected void setLastSkipData(int level) {- super.setLastSkipData(level);- lastDocPointer = docPointer[level];-- if (posPointer != null) {- lastPosPointer = posPointer[level];- lastPosBufferUpto = posBufferUpto[level];- if (payPointer != null) {- lastPayPointer = payPointer[level];- }- if (payloadByteUpto != null) {- lastPayloadByteUpto = payloadByteUpto[level];- }- }- }-- @Override- protected int readSkipData(int level, IndexInput skipStream) throws IOException {- int delta = skipStream.readVInt();- docPointer[level] += skipStream.readVLong();-- if (posPointer != null) {- posPointer[level] += skipStream.readVLong();- posBufferUpto[level] = skipStream.readVInt();-- if (payloadByteUpto != null) {- payloadByteUpto[level] = skipStream.readVInt();- }-- if (payPointer != null) {- payPointer[level] += skipStream.readVLong();- }- }- readImpacts(level, skipStream);- return delta;- }-- // The default impl skips impacts- protected void readImpacts(int level, IndexInput skipStream) throws IOException {- if (version >= Lucene50PostingsFormat.VERSION_IMPACT_SKIP_DATA) {- // The base implementation skips impacts, they are not used- skipStream.skipBytes(skipStream.readVInt());- }- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50StoredFieldsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50StoredFieldsFormat.javadeleted file mode 100644index 560bdb2d1f3..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50StoredFieldsFormat.java+++ /dev/null@@ -1,235 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import java.io.IOException;-import java.util.Objects;-import org.apache.lucene.backward_codecs.compressing.CompressionMode;-import org.apache.lucene.backward_codecs.compressing.Compressor;-import org.apache.lucene.backward_codecs.compressing.Decompressor;-import org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsFormat;-import org.apache.lucene.backward_codecs.packed.LegacyDirectMonotonicWriter;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.StoredFieldsFormat;-import org.apache.lucene.codecs.StoredFieldsReader;-import org.apache.lucene.codecs.StoredFieldsWriter;-import org.apache.lucene.index.CorruptIndexException;-import org.apache.lucene.index.FieldInfos;-import org.apache.lucene.index.SegmentInfo;-import org.apache.lucene.index.StoredFieldVisitor;-import org.apache.lucene.store.DataInput;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.compress.LZ4;--/**- * Lucene 5.0 stored fields format.- *- * <p><b>Principle</b>- *- * <p>This {@link StoredFieldsFormat} compresses blocks of documents in order to improve the- * compression ratio compared to document-level compression. It uses the <a- * href="http://code.google.com/p/lz4/">LZ4</a> compression algorithm by default in 16KB blocks,- * which is fast to compress and very fast to decompress data. Although the default compression- * method that is used ({@link Mode#BEST_SPEED BEST_SPEED}) focuses more on speed than on- * compression ratio, it should provide interesting compression ratios for redundant inputs (such as- * log files, HTML or plain text). For higher compression, you can choose ({@link- * Mode#BEST_COMPRESSION BEST_COMPRESSION}), which uses the <a- * href="http://en.wikipedia.org/wiki/DEFLATE">DEFLATE</a> algorithm with 60KB blocks for a better- * ratio at the expense of slower performance. These two options can be configured like this:- *- * <pre><code class="language-java">- * // the default: for high performance- * indexWriterConfig.setCodec(new Lucene54Codec(Mode.BEST_SPEED));- * // instead for higher performance (but slower):- * // indexWriterConfig.setCodec(new Lucene54Codec(Mode.BEST_COMPRESSION));- * </code></pre>- *- * <p><b>File formats</b>- *- * <p>Stored fields are represented by three files:- *- * <ol>- * <li><a id="field_data"></a>- * <p>A fields data file (extension <code>.fdt</code>). This file stores a compact- * representation of documents in compressed blocks of 16KB or more. When writing a segment,- * documents are appended to an in-memory <code>byte[]</code> buffer. When its size reaches- * 16KB or more, some metadata about the documents is flushed to disk, immediately followed by- * a compressed representation of the buffer using the <a- * href="https://github.com/lz4/lz4">LZ4</a> <a- * href="http://fastcompression.blogspot.fr/2011/05/lz4-explained.html">compression- * format</a>.- * <p>Notes- * <ul>- * <li>If documents are larger than 16KB then chunks will likely contain only one document.- * However, documents can never spread across several chunks (all fields of a single- * document are in the same chunk).- * <li>When at least one document in a chunk is large enough so that the chunk is larger- * than 32KB, the chunk will actually be compressed in several LZ4 blocks of 16KB. This- * allows {@link StoredFieldVisitor}s which are only interested in the first fields of a- * document to not have to decompress 10MB of data if the document is 10MB, but only- * 16KB.- * <li>Given that the original lengths are written in the metadata of the chunk, the- * decompressor can leverage this information to stop decoding as soon as enough data- * has been decompressed.- * <li>In case documents are incompressible, the overhead of the compression format is less- * than 0.5%.- * </ul>- * <li><a id="field_index"></a>- * <p>A fields index file (extension <code>.fdx</code>). This file stores two {@link- * LegacyDirectMonotonicWriter monotonic arrays}, one for the first doc IDs of each block of- * compressed documents, and another one for the corresponding offsets on disk. At search- * time, the array containing doc IDs is binary-searched in order to find the block that- * contains the expected doc ID, and the associated offset on disk is retrieved from the- * second array.- * <li><a id="field_meta"></a>- * <p>A fields meta file (extension <code>.fdm</code>). This file stores metadata about the- * monotonic arrays stored in the index file.- * </ol>- *- * <p><b>Known limitations</b>- *- * <p>This {@link StoredFieldsFormat} does not support individual documents larger than (<code>- * 2<sup>31</sup> - 2<sup>14</sup></code>) bytes.- *- * @lucene.experimental- */-public class Lucene50StoredFieldsFormat extends StoredFieldsFormat {-- /** Configuration option for stored fields. */- public enum Mode {- /** Trade compression ratio for retrieval speed. */- BEST_SPEED,- /** Trade retrieval speed for compression ratio. */- BEST_COMPRESSION- }-- /** Attribute key for compression mode. */- public static final String MODE_KEY = Lucene50StoredFieldsFormat.class.getSimpleName() + ".mode";-- final Mode mode;-- /** Stored fields format with default options */- public Lucene50StoredFieldsFormat() {- this(Mode.BEST_SPEED);- }-- /** Stored fields format with specified mode */- public Lucene50StoredFieldsFormat(Mode mode) {- this.mode = Objects.requireNonNull(mode);- }-- @Override- public final StoredFieldsReader fieldsReader(- Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {- String value = si.getAttribute(MODE_KEY);- if (value == null) {- throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name);- }- Mode mode = Mode.valueOf(value);- return impl(mode).fieldsReader(directory, si, fn, context);- }-- @Override- public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context)- throws IOException {- throw new UnsupportedOperationException("Old codecs may only be used for reading");- }-- StoredFieldsFormat impl(Mode mode) {- switch (mode) {- case BEST_SPEED:- return new Lucene50CompressingStoredFieldsFormat(- "Lucene50StoredFieldsFastData", FAST_MODE, 1 << 14, 128, 10);- case BEST_COMPRESSION:- return new Lucene50CompressingStoredFieldsFormat(- "Lucene50StoredFieldsHighData", CompressionMode.HIGH_COMPRESSION, 61440, 512, 10);- default:- throw new AssertionError();- }- }-- static final CompressionMode FAST_MODE =- new CompressionMode() {-- @Override- public Compressor newCompressor() {- return new LZ4FastCompressor();- }-- @Override- public Decompressor newDecompressor() {- return LZ4_DECOMPRESSOR;- }-- @Override- public String toString() {- return "FAST";- }- };-- private static final class LZ4FastCompressor extends Compressor {-- private final LZ4.FastCompressionHashTable ht;-- LZ4FastCompressor() {- ht = new LZ4.FastCompressionHashTable();- }-- @Override- public void compress(byte[] bytes, int off, int len, DataOutput out) throws IOException {- LZ4.compress(bytes, off, len, EndiannessReverserUtil.wrapDataOutput(out), ht);- }-- @Override- public void close() throws IOException {- // no-op- }- }-- private static final Decompressor LZ4_DECOMPRESSOR =- new Decompressor() {-- @Override- public void decompress(- DataInput in, int originalLength, int offset, int length, BytesRef bytes)- throws IOException {- assert offset + length <= originalLength;- // add 7 padding bytes, this is not necessary but can help decompression run faster- if (bytes.bytes.length < originalLength + 7) {- bytes.bytes = new byte[ArrayUtil.oversize(originalLength + 7, 1)];- }- final int decompressedLength =- LZ4.decompress(- EndiannessReverserUtil.wrapDataInput(in), offset + length, bytes.bytes, 0);- if (decompressedLength > originalLength) {- throw new CorruptIndexException(- "Corrupted: lengths mismatch: " + decompressedLength + " > " + originalLength, in);- }- bytes.offset = offset;- bytes.length = length;- }-- @Override- public Decompressor clone() {- return this;- }- };-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50TermVectorsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50TermVectorsFormat.javadeleted file mode 100644index 5a37dc450af..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50TermVectorsFormat.java+++ /dev/null@@ -1,158 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsFormat;-import org.apache.lucene.backward_codecs.lucene87.Lucene87StoredFieldsFormat;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.TermVectorsFormat;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.util.packed.BlockPackedWriter;-import org.apache.lucene.util.packed.PackedInts;--/**- * Lucene 5.0 {@link TermVectorsFormat term vectors format}.- *- * <p>Very similarly to {@link Lucene87StoredFieldsFormat}, this format is based on compressed- * chunks of data, with document-level granularity so that a document can never span across distinct- * chunks. Moreover, data is made as compact as possible:- *- * <ul>- * <li>textual data is compressed using the very light, <a- * href="http://code.google.com/p/lz4/">LZ4</a> compression algorithm,- * <li>binary data is written using fixed-size blocks of {@link PackedInts packed ints}.- * </ul>- *- * <p>Term vectors are stored using two files- *- * <ul>- * <li>a data file where terms, frequencies, positions, offsets and payloads are stored,- * <li>an index file, loaded into memory, used to locate specific documents in the data file.- * </ul>- *- * Looking up term vectors for any document requires at most 1 disk seek.- *- * <p><b>File formats</b>- *- * <ol>- * <li><a id="vector_data"></a>- * <p>A vector data file (extension <code>.tvd</code>). This file stores terms, frequencies,- * positions, offsets and payloads for every document. Upon writing a new segment, it- * accumulates data into memory until the buffer used to store terms and payloads grows beyond- * 4KB. Then it flushes all metadata, terms and positions to disk using <a- * href="http://code.google.com/p/lz4/">LZ4</a> compression for terms and payloads and {@link- * BlockPackedWriter blocks of packed ints} for positions.- * <p>Here is a more detailed description of the field data file format:- * <ul>- * <li>VectorData (.tvd) --> <Header>, PackedIntsVersion, ChunkSize,- * <Chunk><sup>ChunkCount</sup>, ChunkCount, DirtyChunkCount, Footer- * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}- * <li>PackedIntsVersion --> {@link PackedInts#VERSION_CURRENT} as a {@link- * DataOutput#writeVInt VInt}- * <li>ChunkSize is the number of bytes of terms to accumulate before flushing, as a {@link- * DataOutput#writeVInt VInt}- * <li>ChunkCount is not known in advance and is the number of chunks necessary to store all- * document of the segment- * <li>Chunk --> DocBase, ChunkDocs, < NumFields >, < FieldNums >, <- * FieldNumOffs >, < Flags >, < NumTerms >, < TermLengths >, <- * TermFreqs >, < Positions >, < StartOffsets >, < Lengths >, <- * PayloadLengths >, < TermAndPayloads >- * <li>DocBase is the ID of the first doc of the chunk as a {@link DataOutput#writeVInt- * VInt}- * <li>ChunkDocs is the number of documents in the chunk- * <li>NumFields --> DocNumFields<sup>ChunkDocs</sup>- * <li>DocNumFields is the number of fields for each doc, written as a {@link- * DataOutput#writeVInt VInt} if ChunkDocs==1 and as a {@link PackedInts} array- * otherwise- * <li>FieldNums --> FieldNumDelta<sup>TotalDistincFields</sup>, a delta-encoded list of- * the sorted unique field numbers present in the chunk- * <li>FieldNumOffs --> FieldNumOff<sup>TotalFields</sup>, as a {@link PackedInts} array- * <li>FieldNumOff is the offset of the field number in FieldNums- * <li>TotalFields is the total number of fields (sum of the values of NumFields)- * <li>Flags --> Bit < FieldFlags >- * <li>Bit is a single bit which when true means that fields have the same options for every- * document in the chunk- * <li>FieldFlags --> if Bit==1: Flag<sup>TotalDistinctFields</sup> else- * Flag<sup>TotalFields</sup>- * <li>Flag: a 3-bits int where:- * <ul>- * <li>the first bit means that the field has positions- * <li>the second bit means that the field has offsets- * <li>the third bit means that the field has payloads- * </ul>- * <li>NumTerms --> FieldNumTerms<sup>TotalFields</sup>- * <li>FieldNumTerms: the number of terms for each field, using {@link BlockPackedWriter- * blocks of 64 packed ints}- * <li>TermLengths --> PrefixLength<sup>TotalTerms</sup>- * SuffixLength<sup>TotalTerms</sup>- * <li>TotalTerms: total number of terms (sum of NumTerms)- * <li>PrefixLength: 0 for the first term of a field, the common prefix with the previous- * term otherwise using {@link BlockPackedWriter blocks of 64 packed ints}- * <li>SuffixLength: length of the term minus PrefixLength for every term using {@link- * BlockPackedWriter blocks of 64 packed ints}- * <li>TermFreqs --> TermFreqMinus1<sup>TotalTerms</sup>- * <li>TermFreqMinus1: (frequency - 1) for each term using {@link BlockPackedWriter blocks- * of 64 packed ints}- * <li>Positions --> PositionDelta<sup>TotalPositions</sup>- * <li>TotalPositions is the sum of frequencies of terms of all fields that have positions- * <li>PositionDelta: the absolute position for the first position of a term, and the- * difference with the previous positions for following positions using {@link- * BlockPackedWriter blocks of 64 packed ints}- * <li>StartOffsets --> (AvgCharsPerTerm<sup>TotalDistinctFields</sup>)- * StartOffsetDelta<sup>TotalOffsets</sup>- * <li>TotalOffsets is the sum of frequencies of terms of all fields that have offsets- * <li>AvgCharsPerTerm: average number of chars per term, encoded as a float on 4 bytes.- * They are not present if no field has both positions and offsets enabled.- * <li>StartOffsetDelta: (startOffset - previousStartOffset - AvgCharsPerTerm *- * PositionDelta). previousStartOffset is 0 for the first offset and AvgCharsPerTerm is- * 0 if the field has no positions using {@link BlockPackedWriter blocks of 64 packed- * ints}- * <li>Lengths --> LengthMinusTermLength<sup>TotalOffsets</sup>- * <li>LengthMinusTermLength: (endOffset - startOffset - termLength) using {@link- * BlockPackedWriter blocks of 64 packed ints}- * <li>PayloadLengths --> PayloadLength<sup>TotalPayloads</sup>- * <li>TotalPayloads is the sum of frequencies of terms of all fields that have payloads- * <li>PayloadLength is the payload length encoded using {@link BlockPackedWriter blocks of- * 64 packed ints}- * <li>TermAndPayloads --> LZ4-compressed representation of < FieldTermsAndPayLoads- * ><sup>TotalFields</sup>- * <li>FieldTermsAndPayLoads --> Terms (Payloads)- * <li>Terms: term bytes- * <li>Payloads: payload bytes (if the field has payloads)- * <li>ChunkCount --> the number of chunks in this file- * <li>DirtyChunkCount --> the number of prematurely flushed chunks in this file- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}- * </ul>- * <li><a id="vector_index"></a>- * <p>An index file (extension <code>.tvx</code>).- * <ul>- * <li>VectorIndex (.tvx) --> <Header>, <ChunkIndex>, Footer- * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}- * <li>ChunkIndex: See FieldsIndexWriter- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}- * </ul>- * </ol>- *- * @lucene.experimental- */-public final class Lucene50TermVectorsFormat extends Lucene50CompressingTermVectorsFormat {-- /** Sole constructor. */- public Lucene50TermVectorsFormat() {- super("Lucene50TermVectorsData", "", Lucene50StoredFieldsFormat.FAST_MODE, 1 << 12, 128, 10);- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndex.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndex.javadeleted file mode 100644index 4a4eba771fa..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndex.java+++ /dev/null@@ -1,32 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50.compressing;--import java.io.Closeable;-import java.io.IOException;--abstract class FieldsIndex implements Cloneable, Closeable {-- /** Get the start pointer for the block that contains the given docID. */- abstract long getStartPointer(int docID);-- /** Check the integrity of the index. */- abstract void checkIntegrity() throws IOException;-- @Override- public abstract FieldsIndex clone();-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndexReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndexReader.javadeleted file mode 100644index 9994948795c..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndexReader.java+++ /dev/null@@ -1,142 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50.compressing;--import java.io.IOException;-import java.io.UncheckedIOException;-import java.util.Objects;-import org.apache.lucene.backward_codecs.packed.LegacyDirectMonotonicReader;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.store.RandomAccessInput;-import org.apache.lucene.util.IOUtils;--final class FieldsIndexReader extends FieldsIndex {-- static final int VERSION_START = 0;- static final int VERSION_CURRENT = 0;-- private final int maxDoc;- private final int blockShift;- private final int numChunks;- private final LegacyDirectMonotonicReader.Meta docsMeta;- private final LegacyDirectMonotonicReader.Meta startPointersMeta;- private final IndexInput indexInput;- private final long docsStartPointer,- docsEndPointer,- startPointersStartPointer,- startPointersEndPointer;- private final LegacyDirectMonotonicReader docs, startPointers;- private final long maxPointer;-- FieldsIndexReader(- Directory dir,- String name,- String suffix,- String extension,- String codecName,- byte[] id,- IndexInput metaIn)- throws IOException {- maxDoc = metaIn.readInt();- blockShift = metaIn.readInt();- numChunks = metaIn.readInt();- docsStartPointer = metaIn.readLong();- docsMeta = LegacyDirectMonotonicReader.loadMeta(metaIn, numChunks, blockShift);- docsEndPointer = startPointersStartPointer = metaIn.readLong();- startPointersMeta = LegacyDirectMonotonicReader.loadMeta(metaIn, numChunks, blockShift);- startPointersEndPointer = metaIn.readLong();- maxPointer = metaIn.readLong();-- indexInput =- EndiannessReverserUtil.openInput(- dir, IndexFileNames.segmentFileName(name, suffix, extension), IOContext.DEFAULT);- try {- CodecUtil.checkIndexHeader(- indexInput, codecName + "Idx", VERSION_START, VERSION_CURRENT, id, suffix);- CodecUtil.retrieveChecksum(indexInput);- } catch (Throwable t) {- IOUtils.closeWhileSuppressingExceptions(t, indexInput);- throw t;- }- final RandomAccessInput docsSlice =- indexInput.randomAccessSlice(docsStartPointer, docsEndPointer - docsStartPointer);- final RandomAccessInput startPointersSlice =- indexInput.randomAccessSlice(- startPointersStartPointer, startPointersEndPointer - startPointersStartPointer);- docs = LegacyDirectMonotonicReader.getInstance(docsMeta, docsSlice);- startPointers = LegacyDirectMonotonicReader.getInstance(startPointersMeta, startPointersSlice);- }-- private FieldsIndexReader(FieldsIndexReader other) throws IOException {- maxDoc = other.maxDoc;- numChunks = other.numChunks;- blockShift = other.blockShift;- docsMeta = other.docsMeta;- startPointersMeta = other.startPointersMeta;- indexInput = other.indexInput.clone();- docsStartPointer = other.docsStartPointer;- docsEndPointer = other.docsEndPointer;- startPointersStartPointer = other.startPointersStartPointer;- startPointersEndPointer = other.startPointersEndPointer;- maxPointer = other.maxPointer;- final RandomAccessInput docsSlice =- indexInput.randomAccessSlice(docsStartPointer, docsEndPointer - docsStartPointer);- final RandomAccessInput startPointersSlice =- indexInput.randomAccessSlice(- startPointersStartPointer, startPointersEndPointer - startPointersStartPointer);- docs = LegacyDirectMonotonicReader.getInstance(docsMeta, docsSlice);- startPointers = LegacyDirectMonotonicReader.getInstance(startPointersMeta, startPointersSlice);- }-- @Override- public void close() throws IOException {- indexInput.close();- }-- @Override- long getStartPointer(int docID) {- Objects.checkIndex(docID, maxDoc);- long blockIndex = docs.binarySearch(0, numChunks, docID);- if (blockIndex < 0) {- blockIndex = -2 - blockIndex;- }- return startPointers.get(blockIndex);- }-- @Override- public FieldsIndex clone() {- try {- return new FieldsIndexReader(this);- } catch (IOException e) {- throw new UncheckedIOException(e);- }- }-- public long getMaxPointer() {- return maxPointer;- }-- @Override- void checkIntegrity() throws IOException {- CodecUtil.checksumEntireFile(indexInput);- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/LegacyFieldsIndexReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/LegacyFieldsIndexReader.javadeleted file mode 100644index 2aba9e05332..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/LegacyFieldsIndexReader.java+++ /dev/null@@ -1,185 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50.compressing;--import static org.apache.lucene.util.BitUtil.zigZagDecode;--import java.io.IOException;-import org.apache.lucene.backward_codecs.packed.LegacyPackedInts;-import org.apache.lucene.index.CorruptIndexException;-import org.apache.lucene.index.SegmentInfo;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.packed.PackedInts;--final class LegacyFieldsIndexReader extends FieldsIndex {-- final int maxDoc;- final int[] docBases;- final long[] startPointers;- final int[] avgChunkDocs;- final long[] avgChunkSizes;- final PackedInts.Reader[] docBasesDeltas; // delta from the avg- final PackedInts.Reader[] startPointersDeltas; // delta from the avg-- // It is the responsibility of the caller to close fieldsIndexIn after this constructor- // has been called- LegacyFieldsIndexReader(IndexInput fieldsIndexIn, SegmentInfo si) throws IOException {- maxDoc = si.maxDoc();- int[] docBases = new int[16];- long[] startPointers = new long[16];- int[] avgChunkDocs = new int[16];- long[] avgChunkSizes = new long[16];- PackedInts.Reader[] docBasesDeltas = new PackedInts.Reader[16];- PackedInts.Reader[] startPointersDeltas = new PackedInts.Reader[16];-- final int packedIntsVersion = fieldsIndexIn.readVInt();-- int blockCount = 0;-- for (; ; ) {- final int numChunks = fieldsIndexIn.readVInt();- if (numChunks == 0) {- break;- }- if (blockCount == docBases.length) {- final int newSize = ArrayUtil.oversize(blockCount + 1, 8);- docBases = ArrayUtil.growExact(docBases, newSize);- startPointers = ArrayUtil.growExact(startPointers, newSize);- avgChunkDocs = ArrayUtil.growExact(avgChunkDocs, newSize);- avgChunkSizes = ArrayUtil.growExact(avgChunkSizes, newSize);- docBasesDeltas = ArrayUtil.growExact(docBasesDeltas, newSize);- startPointersDeltas = ArrayUtil.growExact(startPointersDeltas, newSize);- }-- // doc bases- docBases[blockCount] = fieldsIndexIn.readVInt();- avgChunkDocs[blockCount] = fieldsIndexIn.readVInt();- final int bitsPerDocBase = fieldsIndexIn.readVInt();- if (bitsPerDocBase > 32) {- throw new CorruptIndexException(- "Corrupted bitsPerDocBase: " + bitsPerDocBase, fieldsIndexIn);- }- docBasesDeltas[blockCount] =- LegacyPackedInts.getReaderNoHeader(- fieldsIndexIn,- PackedInts.Format.PACKED,- packedIntsVersion,- numChunks,- bitsPerDocBase);-- // start pointers- startPointers[blockCount] = fieldsIndexIn.readVLong();- avgChunkSizes[blockCount] = fieldsIndexIn.readVLong();- final int bitsPerStartPointer = fieldsIndexIn.readVInt();- if (bitsPerStartPointer > 64) {- throw new CorruptIndexException(- "Corrupted bitsPerStartPointer: " + bitsPerStartPointer, fieldsIndexIn);- }- startPointersDeltas[blockCount] =- LegacyPackedInts.getReaderNoHeader(- fieldsIndexIn,- PackedInts.Format.PACKED,- packedIntsVersion,- numChunks,- bitsPerStartPointer);-- ++blockCount;- }-- this.docBases = ArrayUtil.copyOfSubArray(docBases, 0, blockCount);- this.startPointers = ArrayUtil.copyOfSubArray(startPointers, 0, blockCount);- this.avgChunkDocs = ArrayUtil.copyOfSubArray(avgChunkDocs, 0, blockCount);- this.avgChunkSizes = ArrayUtil.copyOfSubArray(avgChunkSizes, 0, blockCount);- this.docBasesDeltas = ArrayUtil.copyOfSubArray(docBasesDeltas, 0, blockCount);- this.startPointersDeltas = ArrayUtil.copyOfSubArray(startPointersDeltas, 0, blockCount);- }-- private int block(int docID) {- int lo = 0, hi = docBases.length - 1;- while (lo <= hi) {- final int mid = (lo + hi) >>> 1;- final int midValue = docBases[mid];- if (midValue == docID) {- return mid;- } else if (midValue < docID) {- lo = mid + 1;- } else {- hi = mid - 1;- }- }- return hi;- }-- private int relativeDocBase(int block, int relativeChunk) {- final int expected = avgChunkDocs[block] * relativeChunk;- final long delta = zigZagDecode(docBasesDeltas[block].get(relativeChunk));- return expected + (int) delta;- }-- private long relativeStartPointer(int block, int relativeChunk) {- final long expected = avgChunkSizes[block] * relativeChunk;- final long delta = zigZagDecode(startPointersDeltas[block].get(relativeChunk));- return expected + delta;- }-- private int relativeChunk(int block, int relativeDoc) {- int lo = 0, hi = docBasesDeltas[block].size() - 1;- while (lo <= hi) {- final int mid = (lo + hi) >>> 1;- final int midValue = relativeDocBase(block, mid);- if (midValue == relativeDoc) {- return mid;- } else if (midValue < relativeDoc) {- lo = mid + 1;- } else {- hi = mid - 1;- }- }- return hi;- }-- @Override- long getStartPointer(int docID) {- if (docID < 0 || docID >= maxDoc) {- throw new IllegalArgumentException("docID out of range [0-" + maxDoc + "]: " + docID);- }- final int block = block(docID);- final int relativeChunk = relativeChunk(block, docID - docBases[block]);- return startPointers[block] + relativeStartPointer(block, relativeChunk);- }-- @Override- public LegacyFieldsIndexReader clone() {- return this;- }-- @Override- public String toString() {- return getClass().getSimpleName() + "(blocks=" + docBases.length + ")";- }-- @Override- public void close() throws IOException {- // nothing to do- }-- @Override- void checkIntegrity() throws IOException {- // nothing to do, the index is checked at open time- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingStoredFieldsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingStoredFieldsFormat.javadeleted file mode 100644index a1be277abc7..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingStoredFieldsFormat.java+++ /dev/null@@ -1,167 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50.compressing;--import java.io.IOException;-import org.apache.lucene.backward_codecs.compressing.CompressionMode;-import org.apache.lucene.backward_codecs.packed.LegacyDirectMonotonicWriter;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.StoredFieldsFormat;-import org.apache.lucene.codecs.StoredFieldsReader;-import org.apache.lucene.codecs.StoredFieldsWriter;-import org.apache.lucene.index.FieldInfos;-import org.apache.lucene.index.MergePolicy;-import org.apache.lucene.index.SegmentInfo;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;--/**- * A {@link StoredFieldsFormat} that compresses documents in chunks in order to improve the- * compression ratio.- *- * <p>For a chunk size of <var>chunkSize</var> bytes, this {@link StoredFieldsFormat} does not- * support documents larger than (<code>2<sup>31</sup> - chunkSize</code>) bytes.- *- * <p>For optimal performance, you should use a {@link MergePolicy} that returns segments that have- * the biggest byte size first.- *- * @lucene.experimental- */-public class Lucene50CompressingStoredFieldsFormat extends StoredFieldsFormat {-- /** format name */- protected final String formatName;-- /** segment suffix */- protected final String segmentSuffix;-- /** compression mode */- protected final CompressionMode compressionMode;-- /** chunk size */- protected final int chunkSize;-- /** max docs per chunk */- protected final int maxDocsPerChunk;-- /** block shift */- protected final int blockShift;-- /**- * Create a new {@link Lucene50CompressingStoredFieldsFormat} with an empty segment suffix.- *- * @see Lucene50CompressingStoredFieldsFormat#Lucene50CompressingStoredFieldsFormat(String,- * String, CompressionMode, int, int, int)- */- public Lucene50CompressingStoredFieldsFormat(- String formatName,- CompressionMode compressionMode,- int chunkSize,- int maxDocsPerChunk,- int blockShift) {- this(formatName, "", compressionMode, chunkSize, maxDocsPerChunk, blockShift);- }-- /**- * Create a new {@link Lucene50CompressingStoredFieldsFormat}.- *- * <p><code>formatName</code> is the name of the format. This name will be used in the file- * formats to perform {@link CodecUtil#checkIndexHeader codec header checks}.- *- * <p><code>segmentSuffix</code> is the segment suffix. This suffix is added to the result file- * name only if it's not the empty string.- *- * <p>The <code>compressionMode</code> parameter allows you to choose between compression- * algorithms that have various compression and decompression speeds so that you can pick the one- * that best fits your indexing and searching throughput. You should never instantiate two {@link- * Lucene50CompressingStoredFieldsFormat}s that have the same name but different {@link- * CompressionMode}s.- *- * <p><code>chunkSize</code> is the minimum byte size of a chunk of documents. A value of <code>1- * </code> can make sense if there is redundancy across fields. <code>maxDocsPerChunk</code> is an- * upperbound on how many docs may be stored in a single chunk. This is to bound the cpu costs for- * highly compressible data.- *- * <p>Higher values of <code>chunkSize</code> should improve the compression ratio but will- * require more memory at indexing time and might make document loading a little slower (depending- * on the size of your OS cache compared to the size of your index).- *- * @param formatName the name of the {@link StoredFieldsFormat}- * @param compressionMode the {@link CompressionMode} to use- * @param chunkSize the minimum number of bytes of a single chunk of stored documents- * @param maxDocsPerChunk the maximum number of documents in a single chunk- * @param blockShift the log in base 2 of number of chunks to store in an index block- * @see CompressionMode- */- public Lucene50CompressingStoredFieldsFormat(- String formatName,- String segmentSuffix,- CompressionMode compressionMode,- int chunkSize,- int maxDocsPerChunk,- int blockShift) {- this.formatName = formatName;- this.segmentSuffix = segmentSuffix;- this.compressionMode = compressionMode;- if (chunkSize < 1) {- throw new IllegalArgumentException("chunkSize must be >= 1");- }- this.chunkSize = chunkSize;- if (maxDocsPerChunk < 1) {- throw new IllegalArgumentException("maxDocsPerChunk must be >= 1");- }- this.maxDocsPerChunk = maxDocsPerChunk;- if (blockShift < LegacyDirectMonotonicWriter.MIN_BLOCK_SHIFT- || blockShift > LegacyDirectMonotonicWriter.MAX_BLOCK_SHIFT) {- throw new IllegalArgumentException(- "blockSize must be in "- + LegacyDirectMonotonicWriter.MIN_BLOCK_SHIFT- + "-"- + LegacyDirectMonotonicWriter.MAX_BLOCK_SHIFT- + ", got "- + blockShift);- }- this.blockShift = blockShift;- }-- @Override- public StoredFieldsReader fieldsReader(- Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {- return new Lucene50CompressingStoredFieldsReader(- directory, si, segmentSuffix, fn, context, formatName, compressionMode);- }-- @Override- public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context)- throws IOException {- throw new UnsupportedOperationException("Old formats can't be used for writing");- }-- @Override- public String toString() {- return getClass().getSimpleName()- + "(compressionMode="- + compressionMode- + ", chunkSize="- + chunkSize- + ", maxDocsPerChunk="- + maxDocsPerChunk- + ", blockShift="- + blockShift- + ")";- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingStoredFieldsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingStoredFieldsReader.javadeleted file mode 100644index edab3b0d8c2..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingStoredFieldsReader.java+++ /dev/null@@ -1,765 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50.compressing;--import java.io.EOFException;-import java.io.IOException;-import java.util.Arrays;-import org.apache.lucene.backward_codecs.compressing.CompressionMode;-import org.apache.lucene.backward_codecs.compressing.Decompressor;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.StoredFieldsReader;-import org.apache.lucene.document.Document;-import org.apache.lucene.index.CorruptIndexException;-import org.apache.lucene.index.FieldInfo;-import org.apache.lucene.index.FieldInfos;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.SegmentInfo;-import org.apache.lucene.index.StoredFieldDataInput;-import org.apache.lucene.index.StoredFieldVisitor;-import org.apache.lucene.store.AlreadyClosedException;-import org.apache.lucene.store.ByteArrayDataInput;-import org.apache.lucene.store.ChecksumIndexInput;-import org.apache.lucene.store.DataInput;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.BitUtil;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.IOUtils;-import org.apache.lucene.util.LongsRef;-import org.apache.lucene.util.packed.PackedInts;--/**- * {@link StoredFieldsReader} impl for {@link Lucene50CompressingStoredFieldsFormat}.- *- * @lucene.experimental- */-public final class Lucene50CompressingStoredFieldsReader extends StoredFieldsReader {-- /** Extension of stored fields file */- public static final String FIELDS_EXTENSION = "fdt";-- /** Extension of stored fields index */- public static final String INDEX_EXTENSION = "fdx";-- /** Extension of stored fields meta */- public static final String META_EXTENSION = "fdm";-- /** Codec name for the index. */- public static final String INDEX_CODEC_NAME = "Lucene85FieldsIndex";-- static final int STRING = 0x00;- static final int BYTE_ARR = 0x01;- static final int NUMERIC_INT = 0x02;- static final int NUMERIC_FLOAT = 0x03;- static final int NUMERIC_LONG = 0x04;- static final int NUMERIC_DOUBLE = 0x05;-- static final int TYPE_BITS = PackedInts.bitsRequired(NUMERIC_DOUBLE);- static final int TYPE_MASK = (int) PackedInts.maxValue(TYPE_BITS);-- static final int VERSION_START = 1;- static final int VERSION_OFFHEAP_INDEX = 2;-- /** Version where all metadata were moved to the meta file. */- static final int VERSION_META = 3;-- /**- * Version where numChunks is explicitly recorded in meta file and a dirty chunk bit is recorded- * in each chunk- */- static final int VERSION_NUM_CHUNKS = 4;-- static final int VERSION_CURRENT = VERSION_NUM_CHUNKS;- static final int META_VERSION_START = 0;-- // for compression of timestamps- static final long SECOND = 1000L;- static final long HOUR = 60 * 60 * SECOND;- static final long DAY = 24 * HOUR;- static final int SECOND_ENCODING = 0x40;- static final int HOUR_ENCODING = 0x80;- static final int DAY_ENCODING = 0xC0;-- private final int version;- private final FieldInfos fieldInfos;- private final FieldsIndex indexReader;- private final long maxPointer;- private final IndexInput fieldsStream;- private final int chunkSize;- private final int packedIntsVersion;- private final CompressionMode compressionMode;- private final Decompressor decompressor;- private final int numDocs;- private final boolean merging;- private final BlockState state;- private boolean closed;-- // used by clone- private Lucene50CompressingStoredFieldsReader(- Lucene50CompressingStoredFieldsReader reader, boolean merging) {- this.version = reader.version;- this.fieldInfos = reader.fieldInfos;- this.fieldsStream = reader.fieldsStream.clone();- this.indexReader = reader.indexReader.clone();- this.maxPointer = reader.maxPointer;- this.chunkSize = reader.chunkSize;- this.packedIntsVersion = reader.packedIntsVersion;- this.compressionMode = reader.compressionMode;- this.decompressor = reader.decompressor.clone();- this.numDocs = reader.numDocs;- this.merging = merging;- this.state = new BlockState();- this.closed = false;- }-- /** Sole constructor. */- public Lucene50CompressingStoredFieldsReader(- Directory d,- SegmentInfo si,- String segmentSuffix,- FieldInfos fn,- IOContext context,- String formatName,- CompressionMode compressionMode)- throws IOException {- this.compressionMode = compressionMode;- final String segment = si.name;- boolean success = false;- fieldInfos = fn;- numDocs = si.maxDoc();-- final String fieldsStreamFN =- IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION);- ChecksumIndexInput metaIn = null;- try {- // Open the data file- fieldsStream = EndiannessReverserUtil.openInput(d, fieldsStreamFN, context);- version =- CodecUtil.checkIndexHeader(- fieldsStream, formatName, VERSION_START, VERSION_CURRENT, si.getId(), segmentSuffix);- assert CodecUtil.indexHeaderLength(formatName, segmentSuffix)- == fieldsStream.getFilePointer();-- if (version >= VERSION_OFFHEAP_INDEX) {- final String metaStreamFN =- IndexFileNames.segmentFileName(segment, segmentSuffix, META_EXTENSION);- metaIn = EndiannessReverserUtil.openChecksumInput(d, metaStreamFN, IOContext.READONCE);- CodecUtil.checkIndexHeader(- metaIn,- INDEX_CODEC_NAME + "Meta",- META_VERSION_START,- version,- si.getId(),- segmentSuffix);- }- if (version >= VERSION_META) {- chunkSize = metaIn.readVInt();- packedIntsVersion = metaIn.readVInt();- } else {- chunkSize = fieldsStream.readVInt();- packedIntsVersion = fieldsStream.readVInt();- }-- decompressor = compressionMode.newDecompressor();- this.merging = false;- this.state = new BlockState();-- // NOTE: data file is too costly to verify checksum against all the bytes on open,- // but for now we at least verify proper structure of the checksum footer: which looks- // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption- // such as file truncation.- CodecUtil.retrieveChecksum(fieldsStream);-- long maxPointer = -1;- FieldsIndex indexReader = null;-- if (version < VERSION_OFFHEAP_INDEX) {- // Load the index into memory- final String indexName = IndexFileNames.segmentFileName(segment, segmentSuffix, "fdx");- try (ChecksumIndexInput indexStream =- EndiannessReverserUtil.openChecksumInput(d, indexName, context)) {- Throwable priorE = null;- try {- assert formatName.endsWith("Data");- final String codecNameIdx =- formatName.substring(0, formatName.length() - "Data".length()) + "Index";- final int version2 =- CodecUtil.checkIndexHeader(- indexStream,- codecNameIdx,- VERSION_START,- VERSION_CURRENT,- si.getId(),- segmentSuffix);- if (version != version2) {- throw new CorruptIndexException(- "Version mismatch between stored fields index and data: "- + version2- + " != "- + version,- indexStream);- }- assert CodecUtil.indexHeaderLength(codecNameIdx, segmentSuffix)- == indexStream.getFilePointer();- indexReader = new LegacyFieldsIndexReader(indexStream, si);- maxPointer = indexStream.readVLong();- } catch (Throwable exception) {- priorE = exception;- } finally {- CodecUtil.checkFooter(indexStream, priorE);- }- }- } else {- FieldsIndexReader fieldsIndexReader =- new FieldsIndexReader(- d, si.name, segmentSuffix, INDEX_EXTENSION, INDEX_CODEC_NAME, si.getId(), metaIn);- indexReader = fieldsIndexReader;- maxPointer = fieldsIndexReader.getMaxPointer();- }-- this.maxPointer = maxPointer;- this.indexReader = indexReader;-- if (version >= VERSION_NUM_CHUNKS) {- // discard num_chunks- metaIn.readVLong();- }- if (version >= VERSION_META) {- // consume dirty chunks/docs stats we wrote- metaIn.readVLong();- metaIn.readVLong();- }-- if (metaIn != null) {- CodecUtil.checkFooter(metaIn, null);- metaIn.close();- }-- success = true;- } catch (Throwable t) {- if (metaIn != null) {- CodecUtil.checkFooter(metaIn, t);- throw new AssertionError("unreachable");- } else {- throw t;- }- } finally {- if (!success) {- IOUtils.closeWhileHandlingException(this, metaIn);- }- }- }-- /**- * @throws AlreadyClosedException if this FieldsReader is closed- */- private void ensureOpen() throws AlreadyClosedException {- if (closed) {- throw new AlreadyClosedException("this FieldsReader is closed");- }- }-- /** Close the underlying {@link IndexInput}s. */- @Override- public void close() throws IOException {- if (!closed) {- IOUtils.close(indexReader, fieldsStream);- closed = true;- }- }-- private static void readField(DataInput in, StoredFieldVisitor visitor, FieldInfo info, int bits)- throws IOException {- switch (bits & TYPE_MASK) {- case BYTE_ARR:- int length = in.readVInt();- visitor.binaryField(info, new StoredFieldDataInput(in, length));- break;- case STRING:- visitor.stringField(info, in.readString());- break;- case NUMERIC_INT:- visitor.intField(info, in.readZInt());- break;- case NUMERIC_FLOAT:- visitor.floatField(info, readZFloat(in));- break;- case NUMERIC_LONG:- visitor.longField(info, readTLong(in));- break;- case NUMERIC_DOUBLE:- visitor.doubleField(info, readZDouble(in));- break;- default:- throw new AssertionError("Unknown type flag: " + Integer.toHexString(bits));- }- }-- private static void skipField(DataInput in, int bits) throws IOException {- switch (bits & TYPE_MASK) {- case BYTE_ARR:- case STRING:- final int length = in.readVInt();- in.skipBytes(length);- break;- case NUMERIC_INT:- in.readZInt();- break;- case NUMERIC_FLOAT:- readZFloat(in);- break;- case NUMERIC_LONG:- readTLong(in);- break;- case NUMERIC_DOUBLE:- readZDouble(in);- break;- default:- throw new AssertionError("Unknown type flag: " + Integer.toHexString(bits));- }- }-- /**- * Reads a float in a variable-length format. Reads between one and five bytes. Small integral- * values typically take fewer bytes.- */- static float readZFloat(DataInput in) throws IOException {- int b = in.readByte() & 0xFF;- if (b == 0xFF) {- // negative value- return Float.intBitsToFloat(in.readInt());- } else if ((b & 0x80) != 0) {- // small integer [-1..125]- return (b & 0x7f) - 1;- } else {- // positive float- int bits = b << 24 | ((in.readShort() & 0xFFFF) << 8) | (in.readByte() & 0xFF);- return Float.intBitsToFloat(bits);- }- }-- /**- * Reads a double in a variable-length format. Reads between one and nine bytes. Small integral- * values typically take fewer bytes.- */- static double readZDouble(DataInput in) throws IOException {- int b = in.readByte() & 0xFF;- if (b == 0xFF) {- // negative value- return Double.longBitsToDouble(in.readLong());- } else if (b == 0xFE) {- // float- return Float.intBitsToFloat(in.readInt());- } else if ((b & 0x80) != 0) {- // small integer [-1..124]- return (b & 0x7f) - 1;- } else {- // positive double- long bits =- ((long) b) << 56- | ((in.readInt() & 0xFFFFFFFFL) << 24)- | ((in.readShort() & 0xFFFFL) << 8)- | (in.readByte() & 0xFFL);- return Double.longBitsToDouble(bits);- }- }-- /**- * Reads a long in a variable-length format. Reads between one andCorePropLo nine bytes. Small- * values typically take fewer bytes.- */- static long readTLong(DataInput in) throws IOException {- int header = in.readByte() & 0xFF;-- long bits = header & 0x1F;- if ((header & 0x20) != 0) {- // continuation bit- bits |= in.readVLong() << 5;- }-- long l = BitUtil.zigZagDecode(bits);-- switch (header & DAY_ENCODING) {- case SECOND_ENCODING:- l *= SECOND;- break;- case HOUR_ENCODING:- l *= HOUR;- break;- case DAY_ENCODING:- l *= DAY;- break;- case 0:- // uncompressed- break;- default:- throw new AssertionError();- }-- return l;- }-- /**- * A serialized document, you need to decode its input in order to get an actual {@link Document}.- */- static class SerializedDocument {-- // the serialized data- final DataInput in;-- // the number of bytes on which the document is encoded- final int length;-- // the number of stored fields- final int numStoredFields;-- private SerializedDocument(DataInput in, int length, int numStoredFields) {- this.in = in;- this.length = length;- this.numStoredFields = numStoredFields;- }- }-- /** Keeps state about the current block of documents. */- private class BlockState {-- private int docBase, chunkDocs;-- // whether the block has been sliced, this happens for large documents- private boolean sliced;-- private long[] offsets = LongsRef.EMPTY_LONGS;- private long[] numStoredFields = LongsRef.EMPTY_LONGS;-- // the start pointer at which you can read the compressed documents- private long startPointer;-- private final BytesRef spare;- private final BytesRef bytes;-- BlockState() {- if (merging) {- spare = new BytesRef();- bytes = new BytesRef();- } else {- spare = bytes = null;- }- }-- boolean contains(int docID) {- return docID >= docBase && docID < docBase + chunkDocs;- }-- /** Reset this block so that it stores state for the block that contains the given doc id. */- void reset(int docID) throws IOException {- boolean success = false;- try {- doReset(docID);- success = true;- } finally {- if (success == false) {- // if the read failed, set chunkDocs to 0 so that it does not- // contain any docs anymore and is not reused. This should help- // get consistent exceptions when trying to get several- // documents which are in the same corrupted block since it will- // force the header to be decoded again- chunkDocs = 0;- }- }- }-- private void doReset(int docID) throws IOException {- docBase = fieldsStream.readVInt();- final int token = fieldsStream.readVInt();- chunkDocs = version >= VERSION_NUM_CHUNKS ? token >>> 2 : token >>> 1;- if (contains(docID) == false || docBase + chunkDocs > numDocs) {- throw new CorruptIndexException(- "Corrupted: docID="- + docID- + ", docBase="- + docBase- + ", chunkDocs="- + chunkDocs- + ", numDocs="- + numDocs,- fieldsStream);- }-- sliced = (token & 1) != 0;-- offsets = ArrayUtil.grow(offsets, chunkDocs + 1);- numStoredFields = ArrayUtil.grow(numStoredFields, chunkDocs);-- if (chunkDocs == 1) {- numStoredFields[0] = fieldsStream.readVInt();- offsets[1] = fieldsStream.readVInt();- } else {- // Number of stored fields per document- final int bitsPerStoredFields = fieldsStream.readVInt();- if (bitsPerStoredFields == 0) {- Arrays.fill(numStoredFields, 0, chunkDocs, fieldsStream.readVInt());- } else if (bitsPerStoredFields > 31) {- throw new CorruptIndexException(- "bitsPerStoredFields=" + bitsPerStoredFields, fieldsStream);- } else {- final PackedInts.ReaderIterator it =- PackedInts.getReaderIteratorNoHeader(- fieldsStream,- PackedInts.Format.PACKED,- packedIntsVersion,- chunkDocs,- bitsPerStoredFields,- 1024);- for (int i = 0; i < chunkDocs; ) {- final LongsRef next = it.next(Integer.MAX_VALUE);- System.arraycopy(next.longs, next.offset, numStoredFields, i, next.length);- i += next.length;- }- }-- // The stream encodes the length of each document and we decode- // it into a list of monotonically increasing offsets- final int bitsPerLength = fieldsStream.readVInt();- if (bitsPerLength == 0) {- final int length = fieldsStream.readVInt();- for (int i = 0; i < chunkDocs; ++i) {- offsets[1 + i] = (1 + i) * (long) length;- }- } else if (bitsPerStoredFields > 31) {- throw new CorruptIndexException("bitsPerLength=" + bitsPerLength, fieldsStream);- } else {- final PackedInts.ReaderIterator it =- PackedInts.getReaderIteratorNoHeader(- fieldsStream,- PackedInts.Format.PACKED,- packedIntsVersion,- chunkDocs,- bitsPerLength,- 1024);- for (int i = 0; i < chunkDocs; ) {- final LongsRef next = it.next(Integer.MAX_VALUE);- System.arraycopy(next.longs, next.offset, offsets, i + 1, next.length);- i += next.length;- }- for (int i = 0; i < chunkDocs; ++i) {- offsets[i + 1] += offsets[i];- }- }-- // Additional validation: only the empty document has a serialized length of 0- for (int i = 0; i < chunkDocs; ++i) {- final long len = offsets[i + 1] - offsets[i];- final long storedFields = numStoredFields[i];- if ((len == 0) != (storedFields == 0)) {- throw new CorruptIndexException(- "length=" + len + ", numStoredFields=" + storedFields, fieldsStream);- }- }- }-- startPointer = fieldsStream.getFilePointer();-- if (merging) {- final int totalLength = Math.toIntExact(offsets[chunkDocs]);- // decompress eagerly- if (sliced) {- bytes.offset = bytes.length = 0;- for (int decompressed = 0; decompressed < totalLength; ) {- final int toDecompress = Math.min(totalLength - decompressed, chunkSize);- decompressor.decompress(fieldsStream, toDecompress, 0, toDecompress, spare);- bytes.bytes = ArrayUtil.grow(bytes.bytes, bytes.length + spare.length);- System.arraycopy(spare.bytes, spare.offset, bytes.bytes, bytes.length, spare.length);- bytes.length += spare.length;- decompressed += toDecompress;- }- } else {- decompressor.decompress(fieldsStream, totalLength, 0, totalLength, bytes);- }- if (bytes.length != totalLength) {- throw new CorruptIndexException(- "Corrupted: expected chunk size = " + totalLength + ", got " + bytes.length,- fieldsStream);- }- }- }-- /**- * Get the serialized representation of the given docID. This docID has to be contained in the- * current block.- */- SerializedDocument document(int docID) throws IOException {- if (contains(docID) == false) {- throw new IllegalArgumentException();- }-- final int index = docID - docBase;- final int offset = Math.toIntExact(offsets[index]);- final int length = Math.toIntExact(offsets[index + 1]) - offset;- final int totalLength = Math.toIntExact(offsets[chunkDocs]);- final int numStoredFields = Math.toIntExact(this.numStoredFields[index]);-- final BytesRef bytes;- if (merging) {- bytes = this.bytes;- } else {- bytes = new BytesRef();- }-- final DataInput documentInput;- if (length == 0) {- // empty- documentInput = new ByteArrayDataInput();- } else if (merging) {- // already decompressed- documentInput = new ByteArrayDataInput(bytes.bytes, bytes.offset + offset, length);- } else if (sliced) {- fieldsStream.seek(startPointer);- decompressor.decompress(- fieldsStream, chunkSize, offset, Math.min(length, chunkSize - offset), bytes);- documentInput =- new DataInput() {-- int decompressed = bytes.length;-- void fillBuffer() throws IOException {- assert decompressed <= length;- if (decompressed == length) {- throw new EOFException();- }- final int toDecompress = Math.min(length - decompressed, chunkSize);- decompressor.decompress(fieldsStream, toDecompress, 0, toDecompress, bytes);- decompressed += toDecompress;- }-- @Override- public byte readByte() throws IOException {- if (bytes.length == 0) {- fillBuffer();- }- --bytes.length;- return bytes.bytes[bytes.offset++];- }-- @Override- public void readBytes(byte[] b, int offset, int len) throws IOException {- while (len > bytes.length) {- System.arraycopy(bytes.bytes, bytes.offset, b, offset, bytes.length);- len -= bytes.length;- offset += bytes.length;- fillBuffer();- }- System.arraycopy(bytes.bytes, bytes.offset, b, offset, len);- bytes.offset += len;- bytes.length -= len;- }-- @Override- public void skipBytes(long numBytes) throws IOException {- if (numBytes < 0) {- throw new IllegalArgumentException("numBytes must be >= 0, got " + numBytes);- }- while (numBytes > bytes.length) {- numBytes -= bytes.length;- fillBuffer();- }- bytes.offset += numBytes;- bytes.length -= numBytes;- }- };- } else {- fieldsStream.seek(startPointer);- decompressor.decompress(fieldsStream, totalLength, offset, length, bytes);- assert bytes.length == length;- documentInput = new ByteArrayDataInput(bytes.bytes, bytes.offset, bytes.length);- }-- return new SerializedDocument(- EndiannessReverserUtil.wrapDataInput(documentInput), length, numStoredFields);- }- }-- SerializedDocument serializedDocument(int docID) throws IOException {- if (state.contains(docID) == false) {- fieldsStream.seek(indexReader.getStartPointer(docID));- state.reset(docID);- }- assert state.contains(docID);- return state.document(docID);- }-- @Override- public void document(int docID, StoredFieldVisitor visitor) throws IOException {-- final SerializedDocument doc = serializedDocument(docID);-- for (int fieldIDX = 0; fieldIDX < doc.numStoredFields; fieldIDX++) {- final long infoAndBits = doc.in.readVLong();- final int fieldNumber = (int) (infoAndBits >>> TYPE_BITS);- final FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber);-- final int bits = (int) (infoAndBits & TYPE_MASK);- assert bits <= NUMERIC_DOUBLE : "bits=" + Integer.toHexString(bits);-- switch (visitor.needsField(fieldInfo)) {- case YES:- readField(doc.in, visitor, fieldInfo, bits);- break;- case NO:- if (fieldIDX- == doc.numStoredFields - 1) { // don't skipField on last field value; treat like STOP- return;- }- skipField(doc.in, bits);- break;- case STOP:- return;- }- }- }-- @Override- public StoredFieldsReader clone() {- ensureOpen();- return new Lucene50CompressingStoredFieldsReader(this, false);- }-- @Override- public StoredFieldsReader getMergeInstance() {- ensureOpen();- return new Lucene50CompressingStoredFieldsReader(this, true);- }-- @Override- public void checkIntegrity() throws IOException {- indexReader.checkIntegrity();- CodecUtil.checksumEntireFile(fieldsStream);- }-- @Override- public String toString() {- return getClass().getSimpleName()- + "(mode="- + compressionMode- + ",chunksize="- + chunkSize- + ")";- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsFormat.javadeleted file mode 100644index 7abf5772577..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsFormat.java+++ /dev/null@@ -1,130 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50.compressing;--import java.io.IOException;-import org.apache.lucene.backward_codecs.compressing.CompressionMode;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.StoredFieldsFormat;-import org.apache.lucene.codecs.TermVectorsFormat;-import org.apache.lucene.codecs.TermVectorsReader;-import org.apache.lucene.codecs.TermVectorsWriter;-import org.apache.lucene.index.FieldInfos;-import org.apache.lucene.index.SegmentInfo;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;--/**- * A {@link TermVectorsFormat} that compresses chunks of documents together in order to improve the- * compression ratio.- *- * @lucene.experimental- */-public class Lucene50CompressingTermVectorsFormat extends TermVectorsFormat {-- /** format name */- protected final String formatName;-- /** segment suffix */- protected final String segmentSuffix;-- /** compression mode */- protected final CompressionMode compressionMode;-- /** chunk size */- protected final int chunkSize;-- /** block size */- protected final int blockSize;-- /** max docs per chunk */- protected final int maxDocsPerChunk;-- /**- * Create a new {@link Lucene50CompressingTermVectorsFormat}.- *- * <p><code>formatName</code> is the name of the format. This name will be used in the file- * formats to perform {@link CodecUtil#checkIndexHeader codec header checks}.- *- * <p>The <code>compressionMode</code> parameter allows you to choose between compression- * algorithms that have various compression and decompression speeds so that you can pick the one- * that best fits your indexing and searching throughput. You should never instantiate two {@link- * Lucene50CompressingTermVectorsFormat}s that have the same name but different {@link- * CompressionMode}s.- *- * <p><code>chunkSize</code> is the minimum byte size of a chunk of documents. Higher values of- * <code>chunkSize</code> should improve the compression ratio but will require more memory at- * indexing time and might make document loading a little slower (depending on the size of your OS- * cache compared to the size of your index).- *- * @param formatName the name of the {@link StoredFieldsFormat}- * @param segmentSuffix a suffix to append to files created by this format- * @param compressionMode the {@link CompressionMode} to use- * @param chunkSize the minimum number of bytes of a single chunk of stored documents- * @param maxDocsPerChunk the maximum number of documents in a single chunk- * @param blockSize the number of chunks to store in an index block.- * @see CompressionMode- */- public Lucene50CompressingTermVectorsFormat(- String formatName,- String segmentSuffix,- CompressionMode compressionMode,- int chunkSize,- int maxDocsPerChunk,- int blockSize) {- this.formatName = formatName;- this.segmentSuffix = segmentSuffix;- this.compressionMode = compressionMode;- if (chunkSize < 1) {- throw new IllegalArgumentException("chunkSize must be >= 1");- }- this.chunkSize = chunkSize;- this.maxDocsPerChunk = maxDocsPerChunk;- if (blockSize < 1) {- throw new IllegalArgumentException("blockSize must be >= 1");- }- this.blockSize = blockSize;- }-- @Override- public final TermVectorsReader vectorsReader(- Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context)- throws IOException {- return new Lucene50CompressingTermVectorsReader(- directory, segmentInfo, segmentSuffix, fieldInfos, context, formatName, compressionMode);- }-- @Override- public TermVectorsWriter vectorsWriter(- Directory directory, SegmentInfo segmentInfo, IOContext context) throws IOException {- throw new UnsupportedOperationException("Old formats can't be used for writing");- }-- @Override- public String toString() {- return getClass().getSimpleName()- + "(compressionMode="- + compressionMode- + ", chunkSize="- + chunkSize- + ", maxDocsPerChunk="- + maxDocsPerChunk- + ", blockSize="- + blockSize- + ")";- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsReader.javadeleted file mode 100644index bee4ea5e308..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsReader.java+++ /dev/null@@ -1,1303 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50.compressing;--import java.io.IOException;-import java.util.Iterator;-import java.util.NoSuchElementException;-import org.apache.lucene.backward_codecs.compressing.CompressionMode;-import org.apache.lucene.backward_codecs.compressing.Decompressor;-import org.apache.lucene.backward_codecs.packed.LegacyPackedInts;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.TermVectorsReader;-import org.apache.lucene.index.BaseTermsEnum;-import org.apache.lucene.index.CorruptIndexException;-import org.apache.lucene.index.FieldInfo;-import org.apache.lucene.index.FieldInfos;-import org.apache.lucene.index.Fields;-import org.apache.lucene.index.ImpactsEnum;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.PostingsEnum;-import org.apache.lucene.index.SegmentInfo;-import org.apache.lucene.index.SlowImpactsEnum;-import org.apache.lucene.index.Terms;-import org.apache.lucene.index.TermsEnum;-import org.apache.lucene.store.AlreadyClosedException;-import org.apache.lucene.store.ByteArrayDataInput;-import org.apache.lucene.store.ChecksumIndexInput;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.IOUtils;-import org.apache.lucene.util.LongsRef;-import org.apache.lucene.util.packed.BlockPackedReaderIterator;-import org.apache.lucene.util.packed.PackedInts;--/**- * {@link TermVectorsReader} for {@link Lucene50CompressingTermVectorsFormat}.- *- * @lucene.experimental- */-public final class Lucene50CompressingTermVectorsReader extends TermVectorsReader {-- static final String VECTORS_EXTENSION = "tvd";- static final String VECTORS_INDEX_EXTENSION = "tvx";- static final String VECTORS_META_EXTENSION = "tvm";- static final String VECTORS_INDEX_CODEC_NAME = "Lucene85TermVectorsIndex";-- static final int VERSION_START = 1;- static final int VERSION_OFFHEAP_INDEX = 2;-- /** Version where all metadata were moved to the meta file. */- static final int VERSION_META = 3;-- /** Version where numChunks is explicitly recorded in meta file */- static final int VERSION_NUM_CHUNKS = 4;-- static final int VERSION_CURRENT = VERSION_NUM_CHUNKS;- static final int META_VERSION_START = 0;-- static final int PACKED_BLOCK_SIZE = 64;-- static final int POSITIONS = 0x01;- static final int OFFSETS = 0x02;- static final int PAYLOADS = 0x04;- static final int FLAGS_BITS = PackedInts.bitsRequired(POSITIONS | OFFSETS | PAYLOADS);-- private final FieldInfos fieldInfos;- final FieldsIndex indexReader;- final IndexInput vectorsStream;- private final int version;- private final int packedIntsVersion;- private final CompressionMode compressionMode;- private final Decompressor decompressor;- private final int chunkSize;- private final int numDocs;- private boolean closed;- private final BlockPackedReaderIterator reader;- private final long maxPointer; // end of the data section-- // used by clone- private Lucene50CompressingTermVectorsReader(Lucene50CompressingTermVectorsReader reader) {- this.fieldInfos = reader.fieldInfos;- this.vectorsStream = reader.vectorsStream.clone();- this.indexReader = reader.indexReader.clone();- this.packedIntsVersion = reader.packedIntsVersion;- this.compressionMode = reader.compressionMode;- this.decompressor = reader.decompressor.clone();- this.chunkSize = reader.chunkSize;- this.numDocs = reader.numDocs;- this.reader =- new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, PACKED_BLOCK_SIZE, 0);- this.version = reader.version;- this.maxPointer = reader.maxPointer;- this.closed = false;- }-- /** Sole constructor. */- public Lucene50CompressingTermVectorsReader(- Directory d,- SegmentInfo si,- String segmentSuffix,- FieldInfos fn,- IOContext context,- String formatName,- CompressionMode compressionMode)- throws IOException {- this.compressionMode = compressionMode;- final String segment = si.name;- boolean success = false;- fieldInfos = fn;- numDocs = si.maxDoc();-- ChecksumIndexInput metaIn = null;- try {- // Open the data file- final String vectorsStreamFN =- IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION);- vectorsStream = EndiannessReverserUtil.openInput(d, vectorsStreamFN, context);- version =- CodecUtil.checkIndexHeader(- vectorsStream, formatName, VERSION_START, VERSION_CURRENT, si.getId(), segmentSuffix);- assert CodecUtil.indexHeaderLength(formatName, segmentSuffix)- == vectorsStream.getFilePointer();-- if (version >= VERSION_OFFHEAP_INDEX) {- final String metaStreamFN =- IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_META_EXTENSION);- metaIn = EndiannessReverserUtil.openChecksumInput(d, metaStreamFN, IOContext.READONCE);- CodecUtil.checkIndexHeader(- metaIn,- VECTORS_INDEX_CODEC_NAME + "Meta",- META_VERSION_START,- version,- si.getId(),- segmentSuffix);- }-- if (version >= VERSION_META) {- packedIntsVersion = metaIn.readVInt();- chunkSize = metaIn.readVInt();- } else {- packedIntsVersion = vectorsStream.readVInt();- chunkSize = vectorsStream.readVInt();- }-- // NOTE: data file is too costly to verify checksum against all the bytes on open,- // but for now we at least verify proper structure of the checksum footer: which looks- // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption- // such as file truncation.- CodecUtil.retrieveChecksum(vectorsStream);-- FieldsIndex indexReader = null;- long maxPointer = -1;-- if (version < VERSION_OFFHEAP_INDEX) {- // Load the index into memory- final String indexName = IndexFileNames.segmentFileName(segment, segmentSuffix, "tvx");- try (ChecksumIndexInput indexStream =- EndiannessReverserUtil.openChecksumInput(d, indexName, context)) {- Throwable priorE = null;- try {- assert formatName.endsWith("Data");- final String codecNameIdx =- formatName.substring(0, formatName.length() - "Data".length()) + "Index";- final int version2 =- CodecUtil.checkIndexHeader(- indexStream,- codecNameIdx,- VERSION_START,- VERSION_CURRENT,- si.getId(),- segmentSuffix);- if (version != version2) {- throw new CorruptIndexException(- "Version mismatch between stored fields index and data: "- + version- + " != "- + version2,- indexStream);- }- assert CodecUtil.indexHeaderLength(codecNameIdx, segmentSuffix)- == indexStream.getFilePointer();- indexReader = new LegacyFieldsIndexReader(indexStream, si);- maxPointer = indexStream.readVLong(); // the end of the data section- } catch (Throwable exception) {- priorE = exception;- } finally {- CodecUtil.checkFooter(indexStream, priorE);- }- }- } else {- FieldsIndexReader fieldsIndexReader =- new FieldsIndexReader(- d,- si.name,- segmentSuffix,- VECTORS_INDEX_EXTENSION,- VECTORS_INDEX_CODEC_NAME,- si.getId(),- metaIn);- indexReader = fieldsIndexReader;- maxPointer = fieldsIndexReader.getMaxPointer();- }-- this.indexReader = indexReader;- this.maxPointer = maxPointer;-- if (version >= VERSION_NUM_CHUNKS) {- // consume num_chunks- metaIn.readVLong();- }- if (version >= VERSION_META) {- // consume dirty chunks/docs stats we wrote- metaIn.readVLong();- metaIn.readVLong();- }-- decompressor = compressionMode.newDecompressor();- this.reader =- new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, PACKED_BLOCK_SIZE, 0);-- if (metaIn != null) {- CodecUtil.checkFooter(metaIn, null);- metaIn.close();- }-- success = true;- } catch (Throwable t) {- if (metaIn != null) {- CodecUtil.checkFooter(metaIn, t);- throw new AssertionError("unreachable");- } else {- throw t;- }- } finally {- if (!success) {- IOUtils.closeWhileHandlingException(this, metaIn);- }- }- }-- /**- * @throws AlreadyClosedException if this TermVectorsReader is closed- */- private void ensureOpen() throws AlreadyClosedException {- if (closed) {- throw new AlreadyClosedException("this FieldsReader is closed");- }- }-- @Override- public void close() throws IOException {- if (!closed) {- IOUtils.close(indexReader, vectorsStream);- closed = true;- }- }-- @Override- public TermVectorsReader clone() {- return new Lucene50CompressingTermVectorsReader(this);- }-- @Override- public Fields get(int doc) throws IOException {- ensureOpen();-- // seek to the right place- {- final long startPointer = indexReader.getStartPointer(doc);- vectorsStream.seek(startPointer);- }-- // decode- // - docBase: first doc ID of the chunk- // - chunkDocs: number of docs of the chunk- final int docBase = vectorsStream.readVInt();- final int chunkDocs = vectorsStream.readVInt();- if (doc < docBase || doc >= docBase + chunkDocs || docBase + chunkDocs > numDocs) {- throw new CorruptIndexException(- "docBase=" + docBase + ",chunkDocs=" + chunkDocs + ",doc=" + doc, vectorsStream);- }-- final int skip; // number of fields to skip- final int numFields; // number of fields of the document we're looking for- final int totalFields; // total number of fields of the chunk (sum for all docs)- if (chunkDocs == 1) {- skip = 0;- numFields = totalFields = vectorsStream.readVInt();- } else {- reader.reset(vectorsStream, chunkDocs);- int sum = 0;- for (int i = docBase; i < doc; ++i) {- sum += reader.next();- }- skip = sum;- numFields = (int) reader.next();- sum += numFields;- for (int i = doc + 1; i < docBase + chunkDocs; ++i) {- sum += reader.next();- }- totalFields = sum;- }-- if (numFields == 0) {- // no vectors- return null;- }-- // read field numbers that have term vectors- final int[] fieldNums;- {- final int token = vectorsStream.readByte() & 0xFF;- assert token != 0; // means no term vectors, cannot happen since we checked for numFields == 0- final int bitsPerFieldNum = token & 0x1F;- int totalDistinctFields = token >>> 5;- if (totalDistinctFields == 0x07) {- totalDistinctFields += vectorsStream.readVInt();- }- ++totalDistinctFields;- final PackedInts.ReaderIterator it =- PackedInts.getReaderIteratorNoHeader(- vectorsStream,- PackedInts.Format.PACKED,- packedIntsVersion,- totalDistinctFields,- bitsPerFieldNum,- 1);- fieldNums = new int[totalDistinctFields];- for (int i = 0; i < totalDistinctFields; ++i) {- fieldNums[i] = (int) it.next();- }- }-- // read field numbers and flags- final int[] fieldNumOffs = new int[numFields];- final PackedInts.Reader flags;- {- final int bitsPerOff = PackedInts.bitsRequired(fieldNums.length - 1);- final PackedInts.Reader allFieldNumOffs =- LegacyPackedInts.getReaderNoHeader(- vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, totalFields, bitsPerOff);- switch (vectorsStream.readVInt()) {- case 0:- final PackedInts.Reader fieldFlags =- LegacyPackedInts.getReaderNoHeader(- vectorsStream,- PackedInts.Format.PACKED,- packedIntsVersion,- fieldNums.length,- FLAGS_BITS);- PackedInts.Mutable f = PackedInts.getMutable(totalFields, FLAGS_BITS, PackedInts.COMPACT);- for (int i = 0; i < totalFields; ++i) {- final int fieldNumOff = (int) allFieldNumOffs.get(i);- assert fieldNumOff >= 0 && fieldNumOff < fieldNums.length;- final int fgs = (int) fieldFlags.get(fieldNumOff);- f.set(i, fgs);- }- flags = f;- break;- case 1:- flags =- LegacyPackedInts.getReaderNoHeader(- vectorsStream,- PackedInts.Format.PACKED,- packedIntsVersion,- totalFields,- FLAGS_BITS);- break;- default:- throw new AssertionError();- }- for (int i = 0; i < numFields; ++i) {- fieldNumOffs[i] = (int) allFieldNumOffs.get(skip + i);- }- }-- // number of terms per field for all fields- final PackedInts.Reader numTerms;- final int totalTerms;- {- final int bitsRequired = vectorsStream.readVInt();- numTerms =- LegacyPackedInts.getReaderNoHeader(- vectorsStream,- PackedInts.Format.PACKED,- packedIntsVersion,- totalFields,- bitsRequired);- int sum = 0;- for (int i = 0; i < totalFields; ++i) {- sum += numTerms.get(i);- }- totalTerms = sum;- }-- // term lengths- int docOff = 0, docLen = 0, totalLen;- final int[] fieldLengths = new int[numFields];- final int[][] prefixLengths = new int[numFields][];- final int[][] suffixLengths = new int[numFields][];- {- reader.reset(vectorsStream, totalTerms);- // skip- int toSkip = 0;- for (int i = 0; i < skip; ++i) {- toSkip += numTerms.get(i);- }- reader.skip(toSkip);- // read prefix lengths- for (int i = 0; i < numFields; ++i) {- final int termCount = (int) numTerms.get(skip + i);- final int[] fieldPrefixLengths = new int[termCount];- prefixLengths[i] = fieldPrefixLengths;- for (int j = 0; j < termCount; ) {- final LongsRef next = reader.next(termCount - j);- for (int k = 0; k < next.length; ++k) {- fieldPrefixLengths[j++] = (int) next.longs[next.offset + k];- }- }- }- reader.skip(totalTerms - reader.ord());-- reader.reset(vectorsStream, totalTerms);- // skip- toSkip = 0;- for (int i = 0; i < skip; ++i) {- for (int j = 0; j < numTerms.get(i); ++j) {- docOff += reader.next();- }- }- for (int i = 0; i < numFields; ++i) {- final int termCount = (int) numTerms.get(skip + i);- final int[] fieldSuffixLengths = new int[termCount];- suffixLengths[i] = fieldSuffixLengths;- for (int j = 0; j < termCount; ) {- final LongsRef next = reader.next(termCount - j);- for (int k = 0; k < next.length; ++k) {- fieldSuffixLengths[j++] = (int) next.longs[next.offset + k];- }- }- fieldLengths[i] = sum(suffixLengths[i]);- docLen += fieldLengths[i];- }- totalLen = docOff + docLen;- for (int i = skip + numFields; i < totalFields; ++i) {- for (int j = 0; j < numTerms.get(i); ++j) {- totalLen += reader.next();- }- }- }-- // term freqs- final int[] termFreqs = new int[totalTerms];- {- reader.reset(vectorsStream, totalTerms);- for (int i = 0; i < totalTerms; ) {- final LongsRef next = reader.next(totalTerms - i);- for (int k = 0; k < next.length; ++k) {- termFreqs[i++] = 1 + (int) next.longs[next.offset + k];- }- }- }-- // total number of positions, offsets and payloads- int totalPositions = 0, totalOffsets = 0, totalPayloads = 0;- for (int i = 0, termIndex = 0; i < totalFields; ++i) {- final int f = (int) flags.get(i);- final int termCount = (int) numTerms.get(i);- for (int j = 0; j < termCount; ++j) {- final int freq = termFreqs[termIndex++];- if ((f & POSITIONS) != 0) {- totalPositions += freq;- }- if ((f & OFFSETS) != 0) {- totalOffsets += freq;- }- if ((f & PAYLOADS) != 0) {- totalPayloads += freq;- }- }- assert i != totalFields - 1 || termIndex == totalTerms : termIndex + " " + totalTerms;- }-- final int[][] positionIndex = positionIndex(skip, numFields, numTerms, termFreqs);- final int[][] positions, startOffsets, lengths;- if (totalPositions > 0) {- positions =- readPositions(- skip,- numFields,- flags,- numTerms,- termFreqs,- POSITIONS,- totalPositions,- positionIndex);- } else {- positions = new int[numFields][];- }-- if (totalOffsets > 0) {- // average number of chars per term- final float[] charsPerTerm = new float[fieldNums.length];- for (int i = 0; i < charsPerTerm.length; ++i) {- charsPerTerm[i] = Float.intBitsToFloat(vectorsStream.readInt());- }- startOffsets =- readPositions(- skip, numFields, flags, numTerms, termFreqs, OFFSETS, totalOffsets, positionIndex);- lengths =- readPositions(- skip, numFields, flags, numTerms, termFreqs, OFFSETS, totalOffsets, positionIndex);-- for (int i = 0; i < numFields; ++i) {- final int[] fStartOffsets = startOffsets[i];- final int[] fPositions = positions[i];- // patch offsets from positions- if (fStartOffsets != null && fPositions != null) {- final float fieldCharsPerTerm = charsPerTerm[fieldNumOffs[i]];- for (int j = 0; j < startOffsets[i].length; ++j) {- fStartOffsets[j] += (int) (fieldCharsPerTerm * fPositions[j]);- }- }- if (fStartOffsets != null) {- final int[] fPrefixLengths = prefixLengths[i];- final int[] fSuffixLengths = suffixLengths[i];- final int[] fLengths = lengths[i];- for (int j = 0, end = (int) numTerms.get(skip + i); j < end; ++j) {- // delta-decode start offsets and patch lengths using term lengths- final int termLength = fPrefixLengths[j] + fSuffixLengths[j];- lengths[i][positionIndex[i][j]] += termLength;- for (int k = positionIndex[i][j] + 1; k < positionIndex[i][j + 1]; ++k) {- fStartOffsets[k] += fStartOffsets[k - 1];- fLengths[k] += termLength;- }- }- }- }- } else {- startOffsets = lengths = new int[numFields][];- }- if (totalPositions > 0) {- // delta-decode positions- for (int i = 0; i < numFields; ++i) {- final int[] fPositions = positions[i];- final int[] fpositionIndex = positionIndex[i];- if (fPositions != null) {- for (int j = 0, end = (int) numTerms.get(skip + i); j < end; ++j) {- // delta-decode start offsets- for (int k = fpositionIndex[j] + 1; k < fpositionIndex[j + 1]; ++k) {- fPositions[k] += fPositions[k - 1];- }- }- }- }- }-- // payload lengths- final int[][] payloadIndex = new int[numFields][];- int totalPayloadLength = 0;- int payloadOff = 0;- int payloadLen = 0;- if (totalPayloads > 0) {- reader.reset(vectorsStream, totalPayloads);- // skip- int termIndex = 0;- for (int i = 0; i < skip; ++i) {- final int f = (int) flags.get(i);- final int termCount = (int) numTerms.get(i);- if ((f & PAYLOADS) != 0) {- for (int j = 0; j < termCount; ++j) {- final int freq = termFreqs[termIndex + j];- for (int k = 0; k < freq; ++k) {- final int l = (int) reader.next();- payloadOff += l;- }- }- }- termIndex += termCount;- }- totalPayloadLength = payloadOff;- // read doc payload lengths- for (int i = 0; i < numFields; ++i) {- final int f = (int) flags.get(skip + i);- final int termCount = (int) numTerms.get(skip + i);- if ((f & PAYLOADS) != 0) {- final int totalFreq = positionIndex[i][termCount];- payloadIndex[i] = new int[totalFreq + 1];- int posIdx = 0;- payloadIndex[i][posIdx] = payloadLen;- for (int j = 0; j < termCount; ++j) {- final int freq = termFreqs[termIndex + j];- for (int k = 0; k < freq; ++k) {- final int payloadLength = (int) reader.next();- payloadLen += payloadLength;- payloadIndex[i][posIdx + 1] = payloadLen;- ++posIdx;- }- }- assert posIdx == totalFreq;- }- termIndex += termCount;- }- totalPayloadLength += payloadLen;- for (int i = skip + numFields; i < totalFields; ++i) {- final int f = (int) flags.get(i);- final int termCount = (int) numTerms.get(i);- if ((f & PAYLOADS) != 0) {- for (int j = 0; j < termCount; ++j) {- final int freq = termFreqs[termIndex + j];- for (int k = 0; k < freq; ++k) {- totalPayloadLength += reader.next();- }- }- }- termIndex += termCount;- }- assert termIndex == totalTerms : termIndex + " " + totalTerms;- }-- // decompress data- final BytesRef suffixBytes = new BytesRef();- decompressor.decompress(- vectorsStream,- totalLen + totalPayloadLength,- docOff + payloadOff,- docLen + payloadLen,- suffixBytes);- suffixBytes.length = docLen;- final BytesRef payloadBytes =- new BytesRef(suffixBytes.bytes, suffixBytes.offset + docLen, payloadLen);-- final int[] fieldFlags = new int[numFields];- for (int i = 0; i < numFields; ++i) {- fieldFlags[i] = (int) flags.get(skip + i);- }-- final int[] fieldNumTerms = new int[numFields];- for (int i = 0; i < numFields; ++i) {- fieldNumTerms[i] = (int) numTerms.get(skip + i);- }-- final int[][] fieldTermFreqs = new int[numFields][];- {- int termIdx = 0;- for (int i = 0; i < skip; ++i) {- termIdx += numTerms.get(i);- }- for (int i = 0; i < numFields; ++i) {- final int termCount = (int) numTerms.get(skip + i);- fieldTermFreqs[i] = new int[termCount];- for (int j = 0; j < termCount; ++j) {- fieldTermFreqs[i][j] = termFreqs[termIdx++];- }- }- }-- assert sum(fieldLengths) == docLen : sum(fieldLengths) + " != " + docLen;-- return new TVFields(- fieldNums,- fieldFlags,- fieldNumOffs,- fieldNumTerms,- fieldLengths,- prefixLengths,- suffixLengths,- fieldTermFreqs,- positionIndex,- positions,- startOffsets,- lengths,- payloadBytes,- payloadIndex,- suffixBytes);- }-- // field -> term index -> position index- private int[][] positionIndex(- int skip, int numFields, PackedInts.Reader numTerms, int[] termFreqs) {- final int[][] positionIndex = new int[numFields][];- int termIndex = 0;- for (int i = 0; i < skip; ++i) {- final int termCount = (int) numTerms.get(i);- termIndex += termCount;- }- for (int i = 0; i < numFields; ++i) {- final int termCount = (int) numTerms.get(skip + i);- positionIndex[i] = new int[termCount + 1];- for (int j = 0; j < termCount; ++j) {- final int freq = termFreqs[termIndex + j];- positionIndex[i][j + 1] = positionIndex[i][j] + freq;- }- termIndex += termCount;- }- return positionIndex;- }-- private int[][] readPositions(- int skip,- int numFields,- PackedInts.Reader flags,- PackedInts.Reader numTerms,- int[] termFreqs,- int flag,- final int totalPositions,- int[][] positionIndex)- throws IOException {- final int[][] positions = new int[numFields][];- reader.reset(vectorsStream, totalPositions);- // skip- int toSkip = 0;- int termIndex = 0;- for (int i = 0; i < skip; ++i) {- final int f = (int) flags.get(i);- final int termCount = (int) numTerms.get(i);- if ((f & flag) != 0) {- for (int j = 0; j < termCount; ++j) {- final int freq = termFreqs[termIndex + j];- toSkip += freq;- }- }- termIndex += termCount;- }- reader.skip(toSkip);- // read doc positions- for (int i = 0; i < numFields; ++i) {- final int f = (int) flags.get(skip + i);- final int termCount = (int) numTerms.get(skip + i);- if ((f & flag) != 0) {- final int totalFreq = positionIndex[i][termCount];- final int[] fieldPositions = new int[totalFreq];- positions[i] = fieldPositions;- for (int j = 0; j < totalFreq; ) {- final LongsRef nextPositions = reader.next(totalFreq - j);- for (int k = 0; k < nextPositions.length; ++k) {- fieldPositions[j++] = (int) nextPositions.longs[nextPositions.offset + k];- }- }- }- termIndex += termCount;- }- reader.skip(totalPositions - reader.ord());- return positions;- }-- private class TVFields extends Fields {-- private final int[] fieldNums, fieldFlags, fieldNumOffs, numTerms, fieldLengths;- private final int[][] prefixLengths,- suffixLengths,- termFreqs,- positionIndex,- positions,- startOffsets,- lengths,- payloadIndex;- private final BytesRef suffixBytes, payloadBytes;-- public TVFields(- int[] fieldNums,- int[] fieldFlags,- int[] fieldNumOffs,- int[] numTerms,- int[] fieldLengths,- int[][] prefixLengths,- int[][] suffixLengths,- int[][] termFreqs,- int[][] positionIndex,- int[][] positions,- int[][] startOffsets,- int[][] lengths,- BytesRef payloadBytes,- int[][] payloadIndex,- BytesRef suffixBytes) {- this.fieldNums = fieldNums;- this.fieldFlags = fieldFlags;- this.fieldNumOffs = fieldNumOffs;- this.numTerms = numTerms;- this.fieldLengths = fieldLengths;- this.prefixLengths = prefixLengths;- this.suffixLengths = suffixLengths;- this.termFreqs = termFreqs;- this.positionIndex = positionIndex;- this.positions = positions;- this.startOffsets = startOffsets;- this.lengths = lengths;- this.payloadBytes = payloadBytes;- this.payloadIndex = payloadIndex;- this.suffixBytes = suffixBytes;- }-- @Override- public Iterator<String> iterator() {- return new Iterator<>() {- int i = 0;-- @Override- public boolean hasNext() {- return i < fieldNumOffs.length;- }-- @Override- public String next() {- if (!hasNext()) {- throw new NoSuchElementException();- }- final int fieldNum = fieldNums[fieldNumOffs[i++]];- return fieldInfos.fieldInfo(fieldNum).name;- }-- @Override- public void remove() {- throw new UnsupportedOperationException();- }- };- }-- @Override- public Terms terms(String field) throws IOException {- final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);- if (fieldInfo == null) {- return null;- }- int idx = -1;- for (int i = 0; i < fieldNumOffs.length; ++i) {- if (fieldNums[fieldNumOffs[i]] == fieldInfo.number) {- idx = i;- break;- }- }-- if (idx == -1 || numTerms[idx] == 0) {- // no term- return null;- }- int fieldOff = 0, fieldLen = -1;- for (int i = 0; i < fieldNumOffs.length; ++i) {- if (i < idx) {- fieldOff += fieldLengths[i];- } else {- fieldLen = fieldLengths[i];- break;- }- }- assert fieldLen >= 0;- return new TVTerms(- numTerms[idx],- fieldFlags[idx],- prefixLengths[idx],- suffixLengths[idx],- termFreqs[idx],- positionIndex[idx],- positions[idx],- startOffsets[idx],- lengths[idx],- payloadIndex[idx],- payloadBytes,- new BytesRef(suffixBytes.bytes, suffixBytes.offset + fieldOff, fieldLen));- }-- @Override- public int size() {- return fieldNumOffs.length;- }- }-- private static class TVTerms extends Terms {-- private final int numTerms, flags;- private final long totalTermFreq;- private final int[] prefixLengths,- suffixLengths,- termFreqs,- positionIndex,- positions,- startOffsets,- lengths,- payloadIndex;- private final BytesRef termBytes, payloadBytes;-- TVTerms(- int numTerms,- int flags,- int[] prefixLengths,- int[] suffixLengths,- int[] termFreqs,- int[] positionIndex,- int[] positions,- int[] startOffsets,- int[] lengths,- int[] payloadIndex,- BytesRef payloadBytes,- BytesRef termBytes) {- this.numTerms = numTerms;- this.flags = flags;- this.prefixLengths = prefixLengths;- this.suffixLengths = suffixLengths;- this.termFreqs = termFreqs;- this.positionIndex = positionIndex;- this.positions = positions;- this.startOffsets = startOffsets;- this.lengths = lengths;- this.payloadIndex = payloadIndex;- this.payloadBytes = payloadBytes;- this.termBytes = termBytes;- long ttf = 0;- for (int tf : termFreqs) {- ttf += tf;- }- this.totalTermFreq = ttf;- }-- @Override- public TermsEnum iterator() throws IOException {- TVTermsEnum termsEnum = new TVTermsEnum();- termsEnum.reset(- numTerms,- flags,- prefixLengths,- suffixLengths,- termFreqs,- positionIndex,- positions,- startOffsets,- lengths,- payloadIndex,- payloadBytes,- new ByteArrayDataInput(termBytes.bytes, termBytes.offset, termBytes.length));- return termsEnum;- }-- @Override- public long size() throws IOException {- return numTerms;- }-- @Override- public long getSumTotalTermFreq() throws IOException {- return totalTermFreq;- }-- @Override- public long getSumDocFreq() throws IOException {- return numTerms;- }-- @Override- public int getDocCount() throws IOException {- return 1;- }-- @Override- public boolean hasFreqs() {- return true;- }-- @Override- public boolean hasOffsets() {- return (flags & OFFSETS) != 0;- }-- @Override- public boolean hasPositions() {- return (flags & POSITIONS) != 0;- }-- @Override- public boolean hasPayloads() {- return (flags & PAYLOADS) != 0;- }- }-- private static class TVTermsEnum extends BaseTermsEnum {-- private int numTerms, startPos, ord;- private int[] prefixLengths,- suffixLengths,- termFreqs,- positionIndex,- positions,- startOffsets,- lengths,- payloadIndex;- private ByteArrayDataInput in;- private BytesRef payloads;- private final BytesRef term;-- private TVTermsEnum() {- term = new BytesRef(16);- }-- void reset(- int numTerms,- int flags,- int[] prefixLengths,- int[] suffixLengths,- int[] termFreqs,- int[] positionIndex,- int[] positions,- int[] startOffsets,- int[] lengths,- int[] payloadIndex,- BytesRef payloads,- ByteArrayDataInput in) {- this.numTerms = numTerms;- this.prefixLengths = prefixLengths;- this.suffixLengths = suffixLengths;- this.termFreqs = termFreqs;- this.positionIndex = positionIndex;- this.positions = positions;- this.startOffsets = startOffsets;- this.lengths = lengths;- this.payloadIndex = payloadIndex;- this.payloads = payloads;- this.in = in;- startPos = in.getPosition();- reset();- }-- void reset() {- term.length = 0;- in.setPosition(startPos);- ord = -1;- }-- @Override- public BytesRef next() throws IOException {- if (ord == numTerms - 1) {- return null;- } else {- assert ord < numTerms;- ++ord;- }-- // read term- term.offset = 0;- term.length = prefixLengths[ord] + suffixLengths[ord];- if (term.length > term.bytes.length) {- term.bytes = ArrayUtil.grow(term.bytes, term.length);- }- in.readBytes(term.bytes, prefixLengths[ord], suffixLengths[ord]);-- return term;- }-- @Override- public SeekStatus seekCeil(BytesRef text) throws IOException {- if (ord < numTerms && ord >= 0) {- final int cmp = term().compareTo(text);- if (cmp == 0) {- return SeekStatus.FOUND;- } else if (cmp > 0) {- reset();- }- }- // linear scan- while (true) {- final BytesRef term = next();- if (term == null) {- return SeekStatus.END;- }- final int cmp = term.compareTo(text);- if (cmp > 0) {- return SeekStatus.NOT_FOUND;- } else if (cmp == 0) {- return SeekStatus.FOUND;- }- }- }-- @Override- public void seekExact(long ord) throws IOException {- throw new UnsupportedOperationException();- }-- @Override- public BytesRef term() throws IOException {- return term;- }-- @Override- public long ord() throws IOException {- throw new UnsupportedOperationException();- }-- @Override- public int docFreq() throws IOException {- return 1;- }-- @Override- public long totalTermFreq() throws IOException {- return termFreqs[ord];- }-- @Override- public final PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException {- final TVPostingsEnum docsEnum;- if (reuse != null && reuse instanceof TVPostingsEnum) {- docsEnum = (TVPostingsEnum) reuse;- } else {- docsEnum = new TVPostingsEnum();- }-- docsEnum.reset(- termFreqs[ord],- positionIndex[ord],- positions,- startOffsets,- lengths,- payloads,- payloadIndex);- return docsEnum;- }-- @Override- public ImpactsEnum impacts(int flags) throws IOException {- final PostingsEnum delegate = postings(null, PostingsEnum.FREQS);- return new SlowImpactsEnum(delegate);- }- }-- private static class TVPostingsEnum extends PostingsEnum {-- private int doc = -1;- private int termFreq;- private int positionIndex;- private int[] positions;- private int[] startOffsets;- private int[] lengths;- private final BytesRef payload;- private int[] payloadIndex;- private int basePayloadOffset;- private int i;-- TVPostingsEnum() {- payload = new BytesRef();- }-- public void reset(- int freq,- int positionIndex,- int[] positions,- int[] startOffsets,- int[] lengths,- BytesRef payloads,- int[] payloadIndex) {- this.termFreq = freq;- this.positionIndex = positionIndex;- this.positions = positions;- this.startOffsets = startOffsets;- this.lengths = lengths;- this.basePayloadOffset = payloads.offset;- this.payload.bytes = payloads.bytes;- payload.offset = payload.length = 0;- this.payloadIndex = payloadIndex;-- doc = i = -1;- }-- private void checkDoc() {- if (doc == NO_MORE_DOCS) {- throw new IllegalStateException("DocsEnum exhausted");- } else if (doc == -1) {- throw new IllegalStateException("DocsEnum not started");- }- }-- private void checkPosition() {- checkDoc();- if (i < 0) {- throw new IllegalStateException("Position enum not started");- } else if (i >= termFreq) {- throw new IllegalStateException("Read past last position");- }- }-- @Override- public int nextPosition() throws IOException {- if (doc != 0) {- throw new IllegalStateException();- } else if (i >= termFreq - 1) {- throw new IllegalStateException("Read past last position");- }-- ++i;-- if (payloadIndex != null) {- payload.offset = basePayloadOffset + payloadIndex[positionIndex + i];- payload.length = payloadIndex[positionIndex + i + 1] - payloadIndex[positionIndex + i];- }-- if (positions == null) {- return -1;- } else {- return positions[positionIndex + i];- }- }-- @Override- public int startOffset() throws IOException {- checkPosition();- if (startOffsets == null) {- return -1;- } else {- return startOffsets[positionIndex + i];- }- }-- @Override- public int endOffset() throws IOException {- checkPosition();- if (startOffsets == null) {- return -1;- } else {- return startOffsets[positionIndex + i] + lengths[positionIndex + i];- }- }-- @Override- public BytesRef getPayload() throws IOException {- checkPosition();- if (payloadIndex == null || payload.length == 0) {- return null;- } else {- return payload;- }- }-- @Override- public int freq() throws IOException {- checkDoc();- return termFreq;- }-- @Override- public int docID() {- return doc;- }-- @Override- public int nextDoc() throws IOException {- if (doc == -1) {- return (doc = 0);- } else {- return (doc = NO_MORE_DOCS);- }- }-- @Override- public int advance(int target) throws IOException {- return slowAdvance(target);- }-- @Override- public long cost() {- return 1;- }- }-- private static int sum(int[] arr) {- int sum = 0;- for (int el : arr) {- sum += el;- }- return sum;- }-- @Override- public void checkIntegrity() throws IOException {- indexReader.checkIntegrity();- CodecUtil.checksumEntireFile(vectorsStream);- }-- @Override- public String toString() {- return getClass().getSimpleName()- + "(mode="- + compressionMode- + ",chunksize="- + chunkSize- + ")";- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/package-info.javadeleted file mode 100644index 53d5f4bd1af..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/package-info.java+++ /dev/null@@ -1,19 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--/** Lucene 5.0 compressing format. */-package org.apache.lucene.backward_codecs.lucene50.compressing;diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/package-info.javadeleted file mode 100644index 6a01149d255..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/package-info.java+++ /dev/null@@ -1,19 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--/** Lucene 5.0 file format. */-package org.apache.lucene.backward_codecs.lucene50;diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60FieldInfosFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60FieldInfosFormat.javadeleted file mode 100644index 4e0d7c8e53e..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60FieldInfosFormat.java+++ /dev/null@@ -1,372 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene60;--import java.io.IOException;-import java.util.Collections;-import java.util.Map;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.DocValuesFormat;-import org.apache.lucene.codecs.FieldInfosFormat;-import org.apache.lucene.index.CorruptIndexException;-import org.apache.lucene.index.DocValuesSkipIndexType;-import org.apache.lucene.index.DocValuesType;-import org.apache.lucene.index.FieldInfo;-import org.apache.lucene.index.FieldInfos;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.IndexOptions;-import org.apache.lucene.index.SegmentInfo;-import org.apache.lucene.index.VectorEncoding;-import org.apache.lucene.index.VectorSimilarityFunction;-import org.apache.lucene.store.ChecksumIndexInput;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.store.IndexOutput;--/**- * Lucene 6.0 Field Infos format.- *- * <p>Field names are stored in the field info file, with suffix <code>.fnm</code>.- *- * <p>FieldInfos (.fnm) --> Header,FieldsCount, <FieldName,FieldNumber,- * FieldBits,DocValuesBits,DocValuesGen,Attributes,DimensionCount,DimensionNumBytes>- * <sup>FieldsCount</sup>,Footer- *- * <p>Data types:- *- * <ul>- * <li>Header --> {@link CodecUtil#checkIndexHeader IndexHeader}- * <li>FieldsCount --> {@link DataOutput#writeVInt VInt}- * <li>FieldName --> {@link DataOutput#writeString String}- * <li>FieldBits, IndexOptions, DocValuesBits --> {@link DataOutput#writeByte Byte}- * <li>FieldNumber, DimensionCount, DimensionNumBytes --> {@link DataOutput#writeInt VInt}- * <li>Attributes --> {@link DataOutput#writeMapOfStrings Map<String,String>}- * <li>DocValuesGen --> {@link DataOutput#writeLong(long) Int64}- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}- * </ul>- *- * Field Descriptions:- *- * <ul>- * <li>FieldsCount: the number of fields in this file.- * <li>FieldName: name of the field as a UTF-8 String.- * <li>FieldNumber: the field's number. Note that unlike previous versions of Lucene, the fields- * are not numbered implicitly by their order in the file, instead explicitly.- * <li>FieldBits: a byte containing field options.- * <ul>- * <li>The low order bit (0x1) is one for fields that have term vectors stored, and zero for- * fields without term vectors.- * <li>If the second lowest order-bit is set (0x2), norms are omitted for the indexed field.- * <li>If the third lowest-order bit is set (0x4), payloads are stored for the indexed- * field.- * </ul>- * <li>IndexOptions: a byte containing index options.- * <ul>- * <li>0: not indexed- * <li>1: indexed as DOCS_ONLY- * <li>2: indexed as DOCS_AND_FREQS- * <li>3: indexed as DOCS_AND_FREQS_AND_POSITIONS- * <li>4: indexed as DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS- * </ul>- * <li>DocValuesBits: a byte containing per-document value types. The type recorded as two- * four-bit integers, with the high-order bits representing <code>norms</code> options, and- * the low-order bits representing {@code DocValues} options. Each four-bit integer can be- * decoded as such:- * <ul>- * <li>0: no DocValues for this field.- * <li>1: NumericDocValues. ({@link DocValuesType#NUMERIC})- * <li>2: BinaryDocValues. ({@code DocValuesType#BINARY})- * <li>3: SortedDocValues. ({@code DocValuesType#SORTED})- * </ul>- * <li>DocValuesGen is the generation count of the field's DocValues. If this is -1, there are no- * DocValues updates to that field. Anything above zero means there are updates stored by- * {@link DocValuesFormat}.- * <li>Attributes: a key-value map of codec-private attributes.- * <li>PointDimensionCount, PointNumBytes: these are non-zero only if the field is indexed as- * points, e.g. using {@link org.apache.lucene.document.LongPoint}- * </ul>- *- * @lucene.experimental- */-public final class Lucene60FieldInfosFormat extends FieldInfosFormat {-- /** Extension of field infos */- static final String EXTENSION = "fnm";-- // Codec header- static final String CODEC_NAME = "Lucene60FieldInfos";- static final int FORMAT_START = 0;- static final int FORMAT_SOFT_DELETES = 1;- static final int FORMAT_SELECTIVE_INDEXING = 2;- static final int FORMAT_CURRENT = FORMAT_SELECTIVE_INDEXING;-- // Field flags- static final byte STORE_TERMVECTOR = 0x1;- static final byte OMIT_NORMS = 0x2;- static final byte STORE_PAYLOADS = 0x4;- static final byte SOFT_DELETES_FIELD = 0x8;-- /** Sole constructor. */- public Lucene60FieldInfosFormat() {}-- @Override- public FieldInfos read(- Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext context)- throws IOException {- final String fileName =- IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);- try (ChecksumIndexInput input =- EndiannessReverserUtil.openChecksumInput(directory, fileName, context)) {- Throwable priorE = null;- FieldInfo[] infos = null;- try {- int version =- CodecUtil.checkIndexHeader(- input,- Lucene60FieldInfosFormat.CODEC_NAME,- Lucene60FieldInfosFormat.FORMAT_START,- Lucene60FieldInfosFormat.FORMAT_CURRENT,- segmentInfo.getId(),- segmentSuffix);-- infos = readFieldInfos(input, version);-- } catch (Throwable exception) {- priorE = exception;- } finally {- CodecUtil.checkFooter(input, priorE);- }- return new FieldInfos(infos);- }- }-- private FieldInfo[] readFieldInfos(IndexInput input, int version) throws IOException {- final int size = input.readVInt(); // read in the size- FieldInfo[] infos = new FieldInfo[size];-- // previous field's attribute map, we share when possible:- Map<String, String> lastAttributes = Collections.emptyMap();-- for (int i = 0; i < size; i++) {- String name = input.readString();- final int fieldNumber = input.readVInt();- if (fieldNumber < 0) {- throw new CorruptIndexException(- "invalid field number for field: " + name + ", fieldNumber=" + fieldNumber, input);- }- byte bits = input.readByte();- boolean storeTermVector = (bits & STORE_TERMVECTOR) != 0;- boolean omitNorms = (bits & OMIT_NORMS) != 0;- boolean storePayloads = (bits & STORE_PAYLOADS) != 0;- boolean isSoftDeletesField = (bits & SOFT_DELETES_FIELD) != 0;-- final IndexOptions indexOptions = getIndexOptions(input, input.readByte());-- // DV Types are packed in one byte- final DocValuesType docValuesType = getDocValuesType(input, input.readByte());- final long dvGen = input.readLong();- Map<String, String> attributes = input.readMapOfStrings();- // just use the last field's map if its the same- if (attributes.equals(lastAttributes)) {- attributes = lastAttributes;- }- lastAttributes = attributes;- int pointDataDimensionCount = input.readVInt();- int pointNumBytes;- int pointIndexDimensionCount = pointDataDimensionCount;- if (pointDataDimensionCount != 0) {- if (version >= Lucene60FieldInfosFormat.FORMAT_SELECTIVE_INDEXING) {- pointIndexDimensionCount = input.readVInt();- }- pointNumBytes = input.readVInt();- } else {- pointNumBytes = 0;- }-- try {- infos[i] =- new FieldInfo(- name,- fieldNumber,- storeTermVector,- omitNorms,- storePayloads,- indexOptions,- docValuesType,- DocValuesSkipIndexType.NONE,- dvGen,- attributes,- pointDataDimensionCount,- pointIndexDimensionCount,- pointNumBytes,- 0,- VectorEncoding.FLOAT32,- VectorSimilarityFunction.EUCLIDEAN,- isSoftDeletesField,- false);- } catch (IllegalStateException e) {- throw new CorruptIndexException(- "invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e);- }- }- return infos;- }-- static {- // We "mirror" DocValues enum values with the constants below; let's try to ensure if we add a- // new DocValuesType while this format is- // still used for writing, we remember to fix this encoding:- assert DocValuesType.values().length == 6;- }-- private static byte docValuesByte(DocValuesType type) {- switch (type) {- case NONE:- return 0;- case NUMERIC:- return 1;- case BINARY:- return 2;- case SORTED:- return 3;- case SORTED_SET:- return 4;- case SORTED_NUMERIC:- return 5;- default:- // BUG- throw new AssertionError("unhandled DocValuesType: " + type);- }- }-- private static DocValuesType getDocValuesType(IndexInput input, byte b) throws IOException {- switch (b) {- case 0:- return DocValuesType.NONE;- case 1:- return DocValuesType.NUMERIC;- case 2:- return DocValuesType.BINARY;- case 3:- return DocValuesType.SORTED;- case 4:- return DocValuesType.SORTED_SET;- case 5:- return DocValuesType.SORTED_NUMERIC;- default:- throw new CorruptIndexException("invalid docvalues byte: " + b, input);- }- }-- static {- // We "mirror" IndexOptions enum values with the constants below; let's try to ensure if we add- // a new IndexOption while this format is- // still used for writing, we remember to fix this encoding:- assert IndexOptions.values().length == 5;- }-- private static byte indexOptionsByte(IndexOptions indexOptions) {- switch (indexOptions) {- case NONE:- return 0;- case DOCS:- return 1;- case DOCS_AND_FREQS:- return 2;- case DOCS_AND_FREQS_AND_POSITIONS:- return 3;- case DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS:- return 4;- default:- // BUG:- throw new AssertionError("unhandled IndexOptions: " + indexOptions);- }- }-- private static IndexOptions getIndexOptions(IndexInput input, byte b) throws IOException {- switch (b) {- case 0:- return IndexOptions.NONE;- case 1:- return IndexOptions.DOCS;- case 2:- return IndexOptions.DOCS_AND_FREQS;- case 3:- return IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;- case 4:- return IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;- default:- // BUG- throw new CorruptIndexException("invalid IndexOptions byte: " + b, input);- }- }-- /**- * Note: although this format is only used on older versions, we need to keep the write logic in- * addition to the read logic. It's possible for field infos on older segments to be written to,- * for example as part of doc values updates.- */- @Override- public void write(- Directory directory,- SegmentInfo segmentInfo,- String segmentSuffix,- FieldInfos infos,- IOContext context)- throws IOException {- final String fileName =- IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);- try (IndexOutput output = EndiannessReverserUtil.createOutput(directory, fileName, context)) {- CodecUtil.writeIndexHeader(- output,- Lucene60FieldInfosFormat.CODEC_NAME,- Lucene60FieldInfosFormat.FORMAT_CURRENT,- segmentInfo.getId(),- segmentSuffix);- output.writeVInt(infos.size());- for (FieldInfo fi : infos) {- fi.checkConsistency();-- output.writeString(fi.name);- output.writeVInt(fi.number);-- byte bits = 0x0;- if (fi.hasTermVectors()) bits |= STORE_TERMVECTOR;- if (fi.omitsNorms()) bits |= OMIT_NORMS;- if (fi.hasPayloads()) bits |= STORE_PAYLOADS;- if (fi.isSoftDeletesField()) bits |= SOFT_DELETES_FIELD;- output.writeByte(bits);-- output.writeByte(indexOptionsByte(fi.getIndexOptions()));-- // pack the DV type and hasNorms in one byte- output.writeByte(docValuesByte(fi.getDocValuesType()));- output.writeLong(fi.getDocValuesGen());- output.writeMapOfStrings(fi.attributes());- output.writeVInt(fi.getPointDimensionCount());- if (fi.getPointDimensionCount() != 0) {- output.writeVInt(fi.getPointIndexDimensionCount());- output.writeVInt(fi.getPointNumBytes());- }- }- CodecUtil.writeFooter(output);- }- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60PointsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60PointsFormat.javadeleted file mode 100644index 6d826d26d07..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60PointsFormat.java+++ /dev/null@@ -1,107 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene60;--import java.io.IOException;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.PointsFormat;-import org.apache.lucene.codecs.PointsReader;-import org.apache.lucene.codecs.PointsWriter;-import org.apache.lucene.index.SegmentReadState;-import org.apache.lucene.index.SegmentWriteState;--/**- * Lucene 6.0 point format, which encodes dimensional values in a block KD-tree structure for fast- * 1D range and N dimensional shape intersection filtering. See <a- * href="https://www.cs.duke.edu/~pankaj/publications/papers/bkd-sstd.pdf">this paper</a> for- * details.- *- * <p>This data structure is written as a series of blocks on disk, with an in-memory perfectly- * balanced binary tree of split values referencing those blocks at the leaves.- *- * <p>The <code>.dim</code> file has both blocks and the index split values, for each field. The- * file starts with {@link CodecUtil#writeIndexHeader}.- *- * <p>The blocks are written like this:- *- * <ul>- * <li>count (vInt)- * <li>delta-docID (vInt) <sup>count</sup> (delta coded docIDs, in sorted order)- * <li>packedValue<sup>count</sup> (the <code>byte[]</code> value of each dimension packed into a- * single <code>byte[]</code>)- * </ul>- *- * <p>After all blocks for a field are written, then the index is written:- *- * <ul>- * <li>numDims (vInt)- * <li>maxPointsInLeafNode (vInt)- * <li>bytesPerDim (vInt)- * <li>count (vInt)- * <li>packed index (byte[])- * </ul>- *- * <p>The packed index uses hierarchical delta and prefix coding to compactly encode the file- * pointer for all leaf blocks, once the tree is traversed, as well as the split dimension and split- * value for each inner node of the tree.- *- * <p>After all fields blocks + index data are written, {@link CodecUtil#writeFooter} writes the- * checksum.- *- * <p>The <code>.dii</code> file records the file pointer in the <code>.dim</code> file where each- * field's index data was written. It starts with {@link CodecUtil#writeIndexHeader}, then has:- *- * <ul>- * <li>fieldCount (vInt)- * <li>(fieldNumber (vInt), fieldFilePointer (vLong))<sup>fieldCount</sup>- * </ul>- *- * <p>After all fields blocks + index data are written, {@link CodecUtil#writeFooter} writes the- * checksum.- *- * @lucene.experimental- */-public class Lucene60PointsFormat extends PointsFormat {-- static final String DATA_CODEC_NAME = "Lucene60PointsFormatData";- static final String META_CODEC_NAME = "Lucene60PointsFormatMeta";-- /** Filename extension for the leaf blocks */- public static final String DATA_EXTENSION = "dim";-- /** Filename extension for the index per field */- public static final String INDEX_EXTENSION = "dii";-- static final int DATA_VERSION_START = 0;- static final int DATA_VERSION_CURRENT = DATA_VERSION_START;-- static final int INDEX_VERSION_START = 0;- static final int INDEX_VERSION_CURRENT = INDEX_VERSION_START;-- /** Sole constructor */- public Lucene60PointsFormat() {}-- @Override- public PointsWriter fieldsWriter(SegmentWriteState state) throws IOException {- throw new UnsupportedOperationException("Old codecs may only be used for reading");- }-- @Override- public PointsReader fieldsReader(SegmentReadState state) throws IOException {- return new Lucene60PointsReader(state);- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60PointsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60PointsReader.javadeleted file mode 100644index 9a7f6bf1547..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60PointsReader.java+++ /dev/null@@ -1,146 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene60;--import java.io.IOException;-import java.util.HashMap;-import java.util.Map;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.PointsReader;-import org.apache.lucene.index.FieldInfo;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.PointValues;-import org.apache.lucene.index.SegmentReadState;-import org.apache.lucene.store.ChecksumIndexInput;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.util.IOUtils;-import org.apache.lucene.util.bkd.BKDReader;--/** Reads point values previously written with Lucene60PointsWriter */-public class Lucene60PointsReader extends PointsReader {- final IndexInput dataIn;- final SegmentReadState readState;- final Map<Integer, PointValues> readers = new HashMap<>();-- /** Sole constructor */- public Lucene60PointsReader(SegmentReadState readState) throws IOException {- this.readState = readState;-- String indexFileName =- IndexFileNames.segmentFileName(- readState.segmentInfo.name,- readState.segmentSuffix,- Lucene60PointsFormat.INDEX_EXTENSION);-- Map<Integer, Long> fieldToFileOffset = new HashMap<>();-- // Read index file- try (ChecksumIndexInput indexIn =- EndiannessReverserUtil.openChecksumInput(- readState.directory, indexFileName, readState.context)) {- Throwable priorE = null;- try {- CodecUtil.checkIndexHeader(- indexIn,- Lucene60PointsFormat.META_CODEC_NAME,- Lucene60PointsFormat.INDEX_VERSION_START,- Lucene60PointsFormat.INDEX_VERSION_CURRENT,- readState.segmentInfo.getId(),- readState.segmentSuffix);- int count = indexIn.readVInt();- for (int i = 0; i < count; i++) {- int fieldNumber = indexIn.readVInt();- long fp = indexIn.readVLong();- fieldToFileOffset.put(fieldNumber, fp);- }- } catch (Throwable t) {- priorE = t;- } finally {- CodecUtil.checkFooter(indexIn, priorE);- }- }-- String dataFileName =- IndexFileNames.segmentFileName(- readState.segmentInfo.name,- readState.segmentSuffix,- Lucene60PointsFormat.DATA_EXTENSION);- boolean success = false;- dataIn = EndiannessReverserUtil.openInput(readState.directory, dataFileName, readState.context);- try {-- CodecUtil.checkIndexHeader(- dataIn,- Lucene60PointsFormat.DATA_CODEC_NAME,- Lucene60PointsFormat.DATA_VERSION_START,- Lucene60PointsFormat.DATA_VERSION_START,- readState.segmentInfo.getId(),- readState.segmentSuffix);-- // NOTE: data file is too costly to verify checksum against all the bytes on open,- // but for now we at least verify proper structure of the checksum footer: which looks- // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption- // such as file truncation.- CodecUtil.retrieveChecksum(dataIn);-- for (Map.Entry<Integer, Long> ent : fieldToFileOffset.entrySet()) {- int fieldNumber = ent.getKey();- long fp = ent.getValue();- dataIn.seek(fp);- PointValues reader = new BKDReader(dataIn, dataIn, dataIn);- readers.put(fieldNumber, reader);- }-- success = true;- } finally {- if (success == false) {- IOUtils.closeWhileHandlingException(this);- }- }- }-- /**- * Returns the underlying {@link PointValues}.- *- * @lucene.internal- */- @Override- public PointValues getValues(String fieldName) {- FieldInfo fieldInfo = readState.fieldInfos.fieldInfo(fieldName);- if (fieldInfo == null) {- throw new IllegalArgumentException("field=\"" + fieldName + "\" is unrecognized");- }- if (fieldInfo.getPointDimensionCount() == 0) {- throw new IllegalArgumentException("field=\"" + fieldName + "\" did not index point values");- }-- return readers.get(fieldInfo.number);- }-- @Override- public void checkIntegrity() throws IOException {- CodecUtil.checksumEntireFile(dataIn);- }-- @Override- public void close() throws IOException {- dataIn.close();- // Free up heap:- readers.clear();- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/package-info.javadeleted file mode 100644index 9d72a7b984f..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/package-info.java+++ /dev/null@@ -1,19 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--/** Lucene 6.0 file format. */-package org.apache.lucene.backward_codecs.lucene60;diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene70/Lucene70SegmentInfoFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene70/Lucene70SegmentInfoFormat.javadeleted file mode 100644index 84635f09c4f..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene70/Lucene70SegmentInfoFormat.java+++ /dev/null@@ -1,323 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene70;--import java.io.IOException;-import java.util.Map;-import java.util.Set;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.SegmentInfoFormat;-import org.apache.lucene.index.CorruptIndexException;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.IndexWriter;-import org.apache.lucene.index.SegmentInfo;-import org.apache.lucene.index.SegmentInfos;-import org.apache.lucene.search.Sort;-import org.apache.lucene.search.SortField;-import org.apache.lucene.search.SortedNumericSelector;-import org.apache.lucene.search.SortedNumericSortField;-import org.apache.lucene.search.SortedSetSelector;-import org.apache.lucene.search.SortedSetSortField;-import org.apache.lucene.store.ChecksumIndexInput;-import org.apache.lucene.store.DataInput;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.util.Version;--/**- * Lucene 7.0 Segment info format.- *- * <p>Files:- *- * <ul>- * <li><code>.si</code>: Header, SegVersion, SegSize, IsCompoundFile, Diagnostics, Files,- * Attributes, IndexSort, Footer- * </ul>- *- * Data types:- *- * <ul>- * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}- * <li>SegSize --> {@link DataOutput#writeInt Int32}- * <li>SegVersion --> {@link DataOutput#writeString String}- * <li>SegMinVersion --> {@link DataOutput#writeString String}- * <li>Files --> {@link DataOutput#writeSetOfStrings Set<String>}- * <li>Diagnostics,Attributes --> {@link DataOutput#writeMapOfStrings Map<String,String>}- * <li>IsCompoundFile --> {@link DataOutput#writeByte Int8}- * <li>IndexSort --> {@link DataOutput#writeVInt Int32} count, followed by {@code count}- * SortField- * <li>SortField --> {@link DataOutput#writeString String} field name, followed by {@link- * DataOutput#writeVInt Int32} sort type ID, followed by {@link DataOutput#writeByte Int8}- * indicating reversed sort, followed by a type-specific encoding of the optional missing- * value- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}- * </ul>- *- * Field Descriptions:- *- * <ul>- * <li>SegVersion is the code version that created the segment.- * <li>SegMinVersion is the minimum code version that contributed documents to the segment.- * <li>SegSize is the number of documents contained in the segment index.- * <li>IsCompoundFile records whether the segment is written as a compound file or not. If this is- * -1, the segment is not a compound file. If it is 1, the segment is a compound file.- * <li>The Diagnostics Map is privately written by {@link IndexWriter}, as a debugging aid, for- * each segment it creates. It includes metadata like the current Lucene version, OS, Java- * version, why the segment was created (merge, flush, addIndexes), etc.- * <li>Files is a list of files referred to by this segment.- * </ul>- *- * @see SegmentInfos- * @lucene.experimental- */-public class Lucene70SegmentInfoFormat extends SegmentInfoFormat {-- /** File extension used to store {@link SegmentInfo}. */- public static final String SI_EXTENSION = "si";-- static final String CODEC_NAME = "Lucene70SegmentInfo";- static final int VERSION_START = 0;- static final int VERSION_CURRENT = VERSION_START;-- /** Sole constructor. */- public Lucene70SegmentInfoFormat() {}-- @Override- public SegmentInfo read(Directory dir, String segment, byte[] segmentID, IOContext context)- throws IOException {- final String fileName =- IndexFileNames.segmentFileName(segment, "", Lucene70SegmentInfoFormat.SI_EXTENSION);- try (ChecksumIndexInput input =- EndiannessReverserUtil.openChecksumInput(dir, fileName, context)) {- Throwable priorE = null;- SegmentInfo si = null;- try {- CodecUtil.checkIndexHeader(- input,- Lucene70SegmentInfoFormat.CODEC_NAME,- Lucene70SegmentInfoFormat.VERSION_START,- Lucene70SegmentInfoFormat.VERSION_CURRENT,- segmentID,- "");- si = parseSegmentInfo(dir, input, segment, segmentID);- } catch (Throwable exception) {- priorE = exception;- } finally {- CodecUtil.checkFooter(input, priorE);- }- return si;- }- }-- private SegmentInfo parseSegmentInfo(- Directory dir, DataInput input, String segment, byte[] segmentID) throws IOException {- final Version version = Version.fromBits(input.readInt(), input.readInt(), input.readInt());- byte hasMinVersion = input.readByte();- final Version minVersion;- switch (hasMinVersion) {- case 0:- minVersion = null;- break;- case 1:- minVersion = Version.fromBits(input.readInt(), input.readInt(), input.readInt());- break;- default:- throw new CorruptIndexException("Illegal boolean value " + hasMinVersion, input);- }-- final int docCount = input.readInt();- if (docCount < 0) {- throw new CorruptIndexException("invalid docCount: " + docCount, input);- }- final boolean isCompoundFile = input.readByte() == SegmentInfo.YES;-- final Map<String, String> diagnostics = input.readMapOfStrings();- final Set<String> files = input.readSetOfStrings();- final Map<String, String> attributes = input.readMapOfStrings();-- int numSortFields = input.readVInt();- Sort indexSort;- if (numSortFields > 0) {- SortField[] sortFields = new SortField[numSortFields];- for (int i = 0; i < numSortFields; i++) {- String fieldName = input.readString();- int sortTypeID = input.readVInt();- SortField.Type sortType;- SortedSetSelector.Type sortedSetSelector = null;- SortedNumericSelector.Type sortedNumericSelector = null;- switch (sortTypeID) {- case 0:- sortType = SortField.Type.STRING;- break;- case 1:- sortType = SortField.Type.LONG;- break;- case 2:- sortType = SortField.Type.INT;- break;- case 3:- sortType = SortField.Type.DOUBLE;- break;- case 4:- sortType = SortField.Type.FLOAT;- break;- case 5:- sortType = SortField.Type.STRING;- byte selector = input.readByte();- if (selector == 0) {- sortedSetSelector = SortedSetSelector.Type.MIN;- } else if (selector == 1) {- sortedSetSelector = SortedSetSelector.Type.MAX;- } else if (selector == 2) {- sortedSetSelector = SortedSetSelector.Type.MIDDLE_MIN;- } else if (selector == 3) {- sortedSetSelector = SortedSetSelector.Type.MIDDLE_MAX;- } else {- throw new CorruptIndexException(- "invalid index SortedSetSelector ID: " + selector, input);- }- break;- case 6:- byte type = input.readByte();- if (type == 0) {- sortType = SortField.Type.LONG;- } else if (type == 1) {- sortType = SortField.Type.INT;- } else if (type == 2) {- sortType = SortField.Type.DOUBLE;- } else if (type == 3) {- sortType = SortField.Type.FLOAT;- } else {- throw new CorruptIndexException(- "invalid index SortedNumericSortField type ID: " + type, input);- }- byte numericSelector = input.readByte();- if (numericSelector == 0) {- sortedNumericSelector = SortedNumericSelector.Type.MIN;- } else if (numericSelector == 1) {- sortedNumericSelector = SortedNumericSelector.Type.MAX;- } else {- throw new CorruptIndexException(- "invalid index SortedNumericSelector ID: " + numericSelector, input);- }- break;- default:- throw new CorruptIndexException(- "invalid index sort field type ID: " + sortTypeID, input);- }- byte b = input.readByte();- boolean reverse;- if (b == 0) {- reverse = true;- } else if (b == 1) {- reverse = false;- } else {- throw new CorruptIndexException("invalid index sort reverse: " + b, input);- }-- Object missingValue;- b = input.readByte();- if (b == 0) {- missingValue = null;- } else {- switch (sortType) {- case STRING:- if (b == 1) {- missingValue = SortField.STRING_LAST;- } else if (b == 2) {- missingValue = SortField.STRING_FIRST;- } else {- throw new CorruptIndexException("invalid missing value flag: " + b, input);- }- break;- case LONG:- if (b != 1) {- throw new CorruptIndexException("invalid missing value flag: " + b, input);- }- missingValue = input.readLong();- break;- case INT:- if (b != 1) {- throw new CorruptIndexException("invalid missing value flag: " + b, input);- }- missingValue = input.readInt();- break;- case DOUBLE:- if (b != 1) {- throw new CorruptIndexException("invalid missing value flag: " + b, input);- }- missingValue = Double.longBitsToDouble(input.readLong());- break;- case FLOAT:- if (b != 1) {- throw new CorruptIndexException("invalid missing value flag: " + b, input);- }- missingValue = Float.intBitsToFloat(input.readInt());- break;- case CUSTOM:- case DOC:- case REWRITEABLE:- case STRING_VAL:- case SCORE:- default:- throw new AssertionError("unhandled sortType=" + sortType);- }- }-- if (sortedSetSelector != null) {- sortFields[i] =- new SortedSetSortField(fieldName, reverse, sortedSetSelector, missingValue);- } else if (sortedNumericSelector != null) {- sortFields[i] =- new SortedNumericSortField(- fieldName, sortType, reverse, sortedNumericSelector, missingValue);- } else {- sortFields[i] = new SortField(fieldName, sortType, reverse, missingValue);- }- }- indexSort = new Sort(sortFields);- } else if (numSortFields < 0) {- throw new CorruptIndexException("invalid index sort field count: " + numSortFields, input);- } else {- indexSort = null;- }-- SegmentInfo si =- new SegmentInfo(- dir,- version,- minVersion,- segment,- docCount,- isCompoundFile,- false,- null,- diagnostics,- segmentID,- attributes,- indexSort);- si.setFiles(files);- return si;- }-- @Override- public void write(Directory dir, SegmentInfo si, IOContext ioContext) throws IOException {- throw new UnsupportedOperationException("Old formats can't be used for writing");- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene70/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene70/package-info.javadeleted file mode 100644index 43b9a0554cf..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene70/package-info.java+++ /dev/null@@ -1,22 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--/**- * Components from the Lucene 7.0 index format. See {@link- * org.apache.lucene.backward_codecs.lucene86} for an overview of the current index format.- */-package org.apache.lucene.backward_codecs.lucene70;diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80Codec.javadeleted file mode 100644index f4fd646a6ea..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80Codec.java+++ /dev/null@@ -1,135 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene80;--import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;-import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;-import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat;-import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat;-import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;-import org.apache.lucene.backward_codecs.lucene60.Lucene60PointsFormat;-import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat;-import org.apache.lucene.codecs.Codec;-import org.apache.lucene.codecs.CompoundFormat;-import org.apache.lucene.codecs.DocValuesFormat;-import org.apache.lucene.codecs.FieldInfosFormat;-import org.apache.lucene.codecs.KnnVectorsFormat;-import org.apache.lucene.codecs.LiveDocsFormat;-import org.apache.lucene.codecs.NormsFormat;-import org.apache.lucene.codecs.PointsFormat;-import org.apache.lucene.codecs.PostingsFormat;-import org.apache.lucene.codecs.SegmentInfoFormat;-import org.apache.lucene.codecs.StoredFieldsFormat;-import org.apache.lucene.codecs.TermVectorsFormat;-import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;-import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;--/**- * Implements the Lucene 8.0 index format.- *- * @see org.apache.lucene.backward_codecs.lucene80 package documentation for file format details.- * @lucene.experimental- */-public class Lucene80Codec extends Codec {- private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat();- private final FieldInfosFormat fieldInfosFormat = new Lucene60FieldInfosFormat();- private final SegmentInfoFormat segmentInfosFormat = new Lucene70SegmentInfoFormat();- private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();- private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();-- private final PostingsFormat postingsFormat =- new PerFieldPostingsFormat() {- @Override- public PostingsFormat getPostingsFormatForField(String field) {- throw new UnsupportedOperationException("Old codecs can't be used for writing");- }- };-- private final DocValuesFormat docValuesFormat =- new PerFieldDocValuesFormat() {- @Override- public DocValuesFormat getDocValuesFormatForField(String field) {- return defaultDVFormat;- }- };- private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene80");-- private final StoredFieldsFormat storedFieldsFormat;-- /** Instantiates a new codec. */- public Lucene80Codec() {- super("Lucene80");- this.storedFieldsFormat = new Lucene50StoredFieldsFormat();- }-- @Override- public final StoredFieldsFormat storedFieldsFormat() {- return storedFieldsFormat;- }-- @Override- public TermVectorsFormat termVectorsFormat() {- return vectorsFormat;- }-- @Override- public final PostingsFormat postingsFormat() {- return postingsFormat;- }-- @Override- public final FieldInfosFormat fieldInfosFormat() {- return fieldInfosFormat;- }-- @Override- public final SegmentInfoFormat segmentInfoFormat() {- return segmentInfosFormat;- }-- @Override- public final LiveDocsFormat liveDocsFormat() {- return liveDocsFormat;- }-- @Override- public final CompoundFormat compoundFormat() {- return compoundFormat;- }-- @Override- public final PointsFormat pointsFormat() {- return new Lucene60PointsFormat();- }-- @Override- public final DocValuesFormat docValuesFormat() {- return docValuesFormat;- }-- private final NormsFormat normsFormat = new Lucene80NormsFormat();-- @Override- public final NormsFormat normsFormat() {- return normsFormat;- }-- @Override- public final KnnVectorsFormat knnVectorsFormat() {- return KnnVectorsFormat.EMPTY;- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80NormsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80NormsFormat.javadeleted file mode 100644index 28b191c2d6b..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80NormsFormat.java+++ /dev/null@@ -1,105 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene80;--import java.io.IOException;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.NormsConsumer;-import org.apache.lucene.codecs.NormsFormat;-import org.apache.lucene.codecs.NormsProducer;-import org.apache.lucene.index.SegmentReadState;-import org.apache.lucene.index.SegmentWriteState;-import org.apache.lucene.store.DataOutput;--/**- * Lucene 8.0 Score normalization format.- *- * <p>Encodes normalization values by encoding each value with the minimum number of bytes needed to- * represent the range (which can be zero).- *- * <p>Files:- *- * <ol>- * <li><code>.nvd</code>: Norms data- * <li><code>.nvm</code>: Norms metadata- * </ol>- *- * <ol>- * <li><a id="nvm"></a>- * <p>The Norms metadata or .nvm file.- * <p>For each norms field, this stores metadata, such as the offset into the Norms data- * (.nvd)- * <p>Norms metadata (.dvm) --> Header,<Entry><sup>NumFields</sup>,Footer- * <ul>- * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}- * <li>Entry --> FieldNumber, DocsWithFieldAddress, DocsWithFieldLength,- * NumDocsWithField, BytesPerNorm, NormsAddress- * <li>FieldNumber --> {@link DataOutput#writeInt Int32}- * <li>DocsWithFieldAddress --> {@link DataOutput#writeLong Int64}- * <li>DocsWithFieldLength --> {@link DataOutput#writeLong Int64}- * <li>NumDocsWithField --> {@link DataOutput#writeInt Int32}- * <li>BytesPerNorm --> {@link DataOutput#writeByte byte}- * <li>NormsAddress --> {@link DataOutput#writeLong Int64}- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}- * </ul>- * <p>FieldNumber of -1 indicates the end of metadata.- * <p>NormsAddress is the pointer to the start of the data in the norms data (.nvd), or the- * singleton value when BytesPerValue = 0. If BytesPerValue is different from 0 then there are- * NumDocsWithField values to read at that offset.- * <p>DocsWithFieldAddress is the pointer to the start of the bit set containing documents- * that have a norm in the norms data (.nvd), or -2 if no documents have a norm value, or -1- * if all documents have a norm value.- * <p>DocsWithFieldLength is the number of bytes used to encode the set of documents that have- * a norm.- * <li><a id="nvd"></a>- * <p>The Norms data or .nvd file.- * <p>For each Norms field, this stores the actual per-document data (the heavy-lifting)- * <p>Norms data (.nvd) --> Header,< Data ><sup>NumFields</sup>,Footer- * <ul>- * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}- * <li>DocsWithFieldData --> {@link IndexedDISI#writeBitSet Bit set of MaxDoc bits}- * <li>NormsData --> {@link DataOutput#writeByte(byte) byte}<sup>NumDocsWithField *- * BytesPerValue</sup>- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}- * </ul>- * </ol>- *- * @lucene.experimental- */-public class Lucene80NormsFormat extends NormsFormat {-- /** Sole Constructor */- public Lucene80NormsFormat() {}-- @Override- public NormsConsumer normsConsumer(SegmentWriteState state) throws IOException {- throw new UnsupportedOperationException("Old codecs may only be used for reading");- }-- @Override- public NormsProducer normsProducer(SegmentReadState state) throws IOException {- return new Lucene80NormsProducer(- state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);- }-- static final String DATA_CODEC = "Lucene80NormsData";- static final String DATA_EXTENSION = "nvd";- static final String METADATA_CODEC = "Lucene80NormsMetadata";- static final String METADATA_EXTENSION = "nvm";- static final int VERSION_START = 0;- static final int VERSION_CURRENT = VERSION_START;-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80NormsProducer.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80NormsProducer.javadeleted file mode 100644index cf4d8df66ee..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80NormsProducer.java+++ /dev/null@@ -1,489 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene80;--import static org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat.VERSION_CURRENT;-import static org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat.VERSION_START;--import java.io.IOException;-import java.util.HashMap;-import java.util.Map;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.NormsProducer;-import org.apache.lucene.index.CorruptIndexException;-import org.apache.lucene.index.DocValues;-import org.apache.lucene.index.FieldInfo;-import org.apache.lucene.index.FieldInfos;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.NumericDocValues;-import org.apache.lucene.index.SegmentReadState;-import org.apache.lucene.store.ChecksumIndexInput;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.store.RandomAccessInput;-import org.apache.lucene.util.IOUtils;--/** Reader for {@link Lucene80NormsFormat} */-final class Lucene80NormsProducer extends NormsProducer implements Cloneable {- // metadata maps (just file pointers and minimal stuff)- private final Map<Integer, NormsEntry> norms = new HashMap<>();- private final int maxDoc;- private IndexInput data;- private boolean merging;- private Map<Integer, IndexInput> disiInputs;- private Map<Integer, RandomAccessInput> disiJumpTables;- private Map<Integer, RandomAccessInput> dataInputs;-- Lucene80NormsProducer(- SegmentReadState state,- String dataCodec,- String dataExtension,- String metaCodec,- String metaExtension)- throws IOException {- maxDoc = state.segmentInfo.maxDoc();- String metaName =- IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);- int version = -1;-- // read in the entries from the metadata file.- try (ChecksumIndexInput in =- EndiannessReverserUtil.openChecksumInput(state.directory, metaName, state.context)) {- Throwable priorE = null;- try {- version =- CodecUtil.checkIndexHeader(- in,- metaCodec,- VERSION_START,- VERSION_CURRENT,- state.segmentInfo.getId(),- state.segmentSuffix);- readFields(in, state.fieldInfos);- } catch (Throwable exception) {- priorE = exception;- } finally {- CodecUtil.checkFooter(in, priorE);- }- }-- String dataName =- IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);- data = EndiannessReverserUtil.openInput(state.directory, dataName, state.context);- boolean success = false;- try {- final int version2 =- CodecUtil.checkIndexHeader(- data,- dataCodec,- VERSION_START,- VERSION_CURRENT,- state.segmentInfo.getId(),- state.segmentSuffix);- if (version != version2) {- throw new CorruptIndexException(- "Format versions mismatch: meta=" + version + ",data=" + version2, data);- }-- // NOTE: data file is too costly to verify checksum against all the bytes on open,- // but for now we at least verify proper structure of the checksum footer: which looks- // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption- // such as file truncation.- CodecUtil.retrieveChecksum(data);-- success = true;- } finally {- if (!success) {- IOUtils.closeWhileHandlingException(this.data);- }- }- }-- @Override- public NormsProducer getMergeInstance() {- Lucene80NormsProducer clone;- try {- clone = (Lucene80NormsProducer) super.clone();- } catch (CloneNotSupportedException e) {- // cannot happen- throw new RuntimeException(e);- }- clone.data = data.clone();- clone.disiInputs = new HashMap<>();- clone.disiJumpTables = new HashMap<>();- clone.dataInputs = new HashMap<>();- clone.merging = true;- return clone;- }-- static class NormsEntry {- byte denseRankPower;- byte bytesPerNorm;- long docsWithFieldOffset;- long docsWithFieldLength;- short jumpTableEntryCount;- int numDocsWithField;- long normsOffset;- }-- abstract static class DenseNormsIterator extends NumericDocValues {-- final int maxDoc;- int doc = -1;-- DenseNormsIterator(int maxDoc) {- this.maxDoc = maxDoc;- }-- @Override- public int docID() {- return doc;- }-- @Override- public int nextDoc() throws IOException {- return advance(doc + 1);- }-- @Override- public int advance(int target) throws IOException {- if (target >= maxDoc) {- return doc = NO_MORE_DOCS;- }- return doc = target;- }-- @Override- public boolean advanceExact(int target) throws IOException {- this.doc = target;- return true;- }-- @Override- public long cost() {- return maxDoc;- }- }-- abstract static class SparseNormsIterator extends NumericDocValues {-- final IndexedDISI disi;-- SparseNormsIterator(IndexedDISI disi) {- this.disi = disi;- }-- @Override- public int docID() {- return disi.docID();- }-- @Override- public int nextDoc() throws IOException {- return disi.nextDoc();- }-- @Override- public int advance(int target) throws IOException {- return disi.advance(target);- }-- @Override- public boolean advanceExact(int target) throws IOException {- return disi.advanceExact(target);- }-- @Override- public long cost() {- return disi.cost();- }- }-- private void readFields(IndexInput meta, FieldInfos infos) throws IOException {- for (int fieldNumber = meta.readInt(); fieldNumber != -1; fieldNumber = meta.readInt()) {- FieldInfo info = infos.fieldInfo(fieldNumber);- if (info == null) {- throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);- } else if (!info.hasNorms()) {- throw new CorruptIndexException("Invalid field: " + info.name, meta);- }- NormsEntry entry = new NormsEntry();- entry.docsWithFieldOffset = meta.readLong();- entry.docsWithFieldLength = meta.readLong();- entry.jumpTableEntryCount = meta.readShort();- entry.denseRankPower = meta.readByte();- entry.numDocsWithField = meta.readInt();- entry.bytesPerNorm = meta.readByte();- switch (entry.bytesPerNorm) {- case 0:- case 1:- case 2:- case 4:- case 8:- break;- default:- throw new CorruptIndexException(- "Invalid bytesPerValue: " + entry.bytesPerNorm + ", field: " + info.name, meta);- }- entry.normsOffset = meta.readLong();- norms.put(info.number, entry);- }- }-- private RandomAccessInput getDataInput(FieldInfo field, NormsEntry entry) throws IOException {- RandomAccessInput slice = null;- if (merging) {- slice = dataInputs.get(field.number);- }- if (slice == null) {- slice =- data.randomAccessSlice(- entry.normsOffset, entry.numDocsWithField * (long) entry.bytesPerNorm);- if (merging) {- dataInputs.put(field.number, slice);- }- }- return slice;- }-- private IndexInput getDisiInput(FieldInfo field, NormsEntry entry) throws IOException {- if (merging == false) {- return IndexedDISI.createBlockSlice(- data,- "docs",- entry.docsWithFieldOffset,- entry.docsWithFieldLength,- entry.jumpTableEntryCount);- }-- IndexInput in = disiInputs.get(field.number);- if (in == null) {- in =- IndexedDISI.createBlockSlice(- data,- "docs",- entry.docsWithFieldOffset,- entry.docsWithFieldLength,- entry.jumpTableEntryCount);- disiInputs.put(field.number, in);- }-- final IndexInput inF = in; // same as in but final-- // Wrap so that reads can be interleaved from the same thread if two- // norms instances are pulled and consumed in parallel. Merging usually- // doesn't need this feature but CheckIndex might, plus we need merge- // instances to behave well and not be trappy.- return new IndexInput("docs") {-- long offset = 0;-- @Override- public void readBytes(byte[] b, int off, int len) throws IOException {- inF.seek(offset);- offset += len;- inF.readBytes(b, off, len);- }-- @Override- public byte readByte() throws IOException {- throw new UnsupportedOperationException("Unused by IndexedDISI");- }-- @Override- public IndexInput slice(String sliceDescription, long offset, long length)- throws IOException {- throw new UnsupportedOperationException("Unused by IndexedDISI");- }-- @Override- public short readShort() throws IOException {- inF.seek(offset);- offset += Short.BYTES;- return inF.readShort();- }-- @Override- public long readLong() throws IOException {- inF.seek(offset);- offset += Long.BYTES;- return inF.readLong();- }-- @Override- public void seek(long pos) throws IOException {- offset = pos;- }-- @Override- public long length() {- throw new UnsupportedOperationException("Unused by IndexedDISI");- }-- @Override- public long getFilePointer() {- return offset;- }-- @Override- public void close() throws IOException {- throw new UnsupportedOperationException("Unused by IndexedDISI");- }- };- }-- private RandomAccessInput getDisiJumpTable(FieldInfo field, NormsEntry entry) throws IOException {- RandomAccessInput jumpTable = null;- if (merging) {- jumpTable = disiJumpTables.get(field.number);- }- if (jumpTable == null) {- jumpTable =- IndexedDISI.createJumpTable(- data,- entry.docsWithFieldOffset,- entry.docsWithFieldLength,- entry.jumpTableEntryCount);- if (merging) {- disiJumpTables.put(field.number, jumpTable);- }- }- return jumpTable;- }-- @Override- public NumericDocValues getNorms(FieldInfo field) throws IOException {- final NormsEntry entry = norms.get(field.number);- if (entry.docsWithFieldOffset == -2) {- // empty- return DocValues.emptyNumeric();- } else if (entry.docsWithFieldOffset == -1) {- // dense- if (entry.bytesPerNorm == 0) {- return new DenseNormsIterator(maxDoc) {- @Override- public long longValue() throws IOException {- return entry.normsOffset;- }- };- }- final RandomAccessInput slice = getDataInput(field, entry);- switch (entry.bytesPerNorm) {- case 1:- return new DenseNormsIterator(maxDoc) {- @Override- public long longValue() throws IOException {- return slice.readByte(doc);- }- };- case 2:- return new DenseNormsIterator(maxDoc) {- @Override- public long longValue() throws IOException {- return slice.readShort(((long) doc) << 1);- }- };- case 4:- return new DenseNormsIterator(maxDoc) {- @Override- public long longValue() throws IOException {- return slice.readInt(((long) doc) << 2);- }- };- case 8:- return new DenseNormsIterator(maxDoc) {- @Override- public long longValue() throws IOException {- return slice.readLong(((long) doc) << 3);- }- };- default:- // should not happen, we already validate bytesPerNorm in readFields- throw new AssertionError();- }- } else {- // sparse- final IndexInput disiInput = getDisiInput(field, entry);- final RandomAccessInput disiJumpTable = getDisiJumpTable(field, entry);- final IndexedDISI disi =- new IndexedDISI(- disiInput,- disiJumpTable,- entry.jumpTableEntryCount,- entry.denseRankPower,- entry.numDocsWithField);-- if (entry.bytesPerNorm == 0) {- return new SparseNormsIterator(disi) {- @Override- public long longValue() throws IOException {- return entry.normsOffset;- }- };- }- final RandomAccessInput slice = getDataInput(field, entry);- switch (entry.bytesPerNorm) {- case 1:- return new SparseNormsIterator(disi) {- @Override- public long longValue() throws IOException {- return slice.readByte(disi.index());- }- };- case 2:- return new SparseNormsIterator(disi) {- @Override- public long longValue() throws IOException {- return slice.readShort(((long) disi.index()) << 1);- }- };- case 4:- return new SparseNormsIterator(disi) {- @Override- public long longValue() throws IOException {- return slice.readInt(((long) disi.index()) << 2);- }- };- case 8:- return new SparseNormsIterator(disi) {- @Override- public long longValue() throws IOException {- return slice.readLong(((long) disi.index()) << 3);- }- };- default:- // should not happen, we already validate bytesPerNorm in readFields- throw new AssertionError();- }- }- }-- @Override- public void close() throws IOException {- data.close();- }-- @Override- public void checkIntegrity() throws IOException {- CodecUtil.checksumEntireFile(data);- }-- @Override- public String toString() {- return getClass().getSimpleName() + "(fields=" + norms.size() + ")";- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/ForDeltaUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/ForDeltaUtil.javadeleted file mode 100644index dd5f1f7a6a3..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/ForDeltaUtil.java+++ /dev/null@@ -1,86 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene84;--import java.io.IOException;-import org.apache.lucene.store.DataInput;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.util.packed.PackedInts;--/** Utility class to encode/decode increasing sequences of 128 integers. */-public class ForDeltaUtil {-- // IDENTITY_PLUS_ONE[i] == i+1- private static final long[] IDENTITY_PLUS_ONE = new long[ForUtil.BLOCK_SIZE];-- static {- for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {- IDENTITY_PLUS_ONE[i] = i + 1;- }- }-- private static void prefixSumOfOnes(long[] arr, long base) {- System.arraycopy(IDENTITY_PLUS_ONE, 0, arr, 0, ForUtil.BLOCK_SIZE);- // This loop gets auto-vectorized- for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {- arr[i] += base;- }- }-- private final ForUtil forUtil;-- ForDeltaUtil(ForUtil forUtil) {- this.forUtil = forUtil;- }-- /**- * Encode deltas of a strictly monotonically increasing sequence of integers. The provided {@code- * longs} are expected to be deltas between consecutive values.- */- void encodeDeltas(long[] longs, DataOutput out) throws IOException {- if (longs[0] == 1 && PForUtil.allEqual(longs)) { // happens with very dense postings- out.writeByte((byte) 0);- } else {- long or = 0;- for (long l : longs) {- or |= l;- }- assert or != 0;- final int bitsPerValue = PackedInts.bitsRequired(or);- out.writeByte((byte) bitsPerValue);- forUtil.encode(longs, bitsPerValue, out);- }- }-- /** Decode deltas, compute the prefix sum and add {@code base} to all decoded longs. */- void decodeAndPrefixSum(DataInput in, long base, long[] longs) throws IOException {- final int bitsPerValue = Byte.toUnsignedInt(in.readByte());- if (bitsPerValue == 0) {- prefixSumOfOnes(longs, base);- } else {- forUtil.decodeAndPrefixSum(bitsPerValue, in, base, longs);- }- }-- /** Skip a sequence of 128 longs. */- void skip(DataInput in) throws IOException {- final int bitsPerValue = Byte.toUnsignedInt(in.readByte());- if (bitsPerValue != 0) {- in.skipBytes(forUtil.numBytes(bitsPerValue));- }- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/ForUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/ForUtil.javadeleted file mode 100644index f7d1f6c501b..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/ForUtil.java+++ /dev/null@@ -1,1159 +0,0 @@-// This file has been automatically generated, DO NOT EDIT--/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene84;--import java.io.IOException;-import org.apache.lucene.store.DataInput;-import org.apache.lucene.store.DataOutput;--// Inspired from https://fulmicoton.com/posts/bitpacking/-// Encodes multiple integers in a long to get SIMD-like speedups.-// If bitsPerValue <= 8 then we pack 8 ints per long-// else if bitsPerValue <= 16 we pack 4 ints per long-// else we pack 2 ints per long-final class ForUtil {-- static final int BLOCK_SIZE = 128;- private static final int BLOCK_SIZE_LOG2 = 7;-- private static long expandMask32(long mask32) {- return mask32 | (mask32 << 32);- }-- private static long expandMask16(long mask16) {- return expandMask32(mask16 | (mask16 << 16));- }-- private static long expandMask8(long mask8) {- return expandMask16(mask8 | (mask8 << 8));- }-- private static long mask32(int bitsPerValue) {- return expandMask32((1L << bitsPerValue) - 1);- }-- private static long mask16(int bitsPerValue) {- return expandMask16((1L << bitsPerValue) - 1);- }-- private static long mask8(int bitsPerValue) {- return expandMask8((1L << bitsPerValue) - 1);- }-- private static void expand8(long[] arr) {- for (int i = 0; i < 16; ++i) {- long l = arr[i];- arr[i] = (l >>> 56) & 0xFFL;- arr[16 + i] = (l >>> 48) & 0xFFL;- arr[32 + i] = (l >>> 40) & 0xFFL;- arr[48 + i] = (l >>> 32) & 0xFFL;- arr[64 + i] = (l >>> 24) & 0xFFL;- arr[80 + i] = (l >>> 16) & 0xFFL;- arr[96 + i] = (l >>> 8) & 0xFFL;- arr[112 + i] = l & 0xFFL;- }- }-- private static void expand8To32(long[] arr) {- for (int i = 0; i < 16; ++i) {- long l = arr[i];- arr[i] = (l >>> 24) & 0x000000FF000000FFL;- arr[16 + i] = (l >>> 16) & 0x000000FF000000FFL;- arr[32 + i] = (l >>> 8) & 0x000000FF000000FFL;- arr[48 + i] = l & 0x000000FF000000FFL;- }- }-- private static void collapse8(long[] arr) {- for (int i = 0; i < 16; ++i) {- arr[i] =- (arr[i] << 56)- | (arr[16 + i] << 48)- | (arr[32 + i] << 40)- | (arr[48 + i] << 32)- | (arr[64 + i] << 24)- | (arr[80 + i] << 16)- | (arr[96 + i] << 8)- | arr[112 + i];- }- }-- private static void expand16(long[] arr) {- for (int i = 0; i < 32; ++i) {- long l = arr[i];- arr[i] = (l >>> 48) & 0xFFFFL;- arr[32 + i] = (l >>> 32) & 0xFFFFL;- arr[64 + i] = (l >>> 16) & 0xFFFFL;- arr[96 + i] = l & 0xFFFFL;- }- }-- private static void expand16To32(long[] arr) {- for (int i = 0; i < 32; ++i) {- long l = arr[i];- arr[i] = (l >>> 16) & 0x0000FFFF0000FFFFL;- arr[32 + i] = l & 0x0000FFFF0000FFFFL;- }- }-- private static void collapse16(long[] arr) {- for (int i = 0; i < 32; ++i) {- arr[i] = (arr[i] << 48) | (arr[32 + i] << 32) | (arr[64 + i] << 16) | arr[96 + i];- }- }-- private static void expand32(long[] arr) {- for (int i = 0; i < 64; ++i) {- long l = arr[i];- arr[i] = l >>> 32;- arr[64 + i] = l & 0xFFFFFFFFL;- }- }-- private static void collapse32(long[] arr) {- for (int i = 0; i < 64; ++i) {- arr[i] = (arr[i] << 32) | arr[64 + i];- }- }-- private static void prefixSum8(long[] arr, long base) {- expand8To32(arr);- prefixSum32(arr, base);- }-- private static void prefixSum16(long[] arr, long base) {- // We need to move to the next primitive size to avoid overflows- expand16To32(arr);- prefixSum32(arr, base);- }-- private static void prefixSum32(long[] arr, long base) {- arr[0] += base << 32;- innerPrefixSum32(arr);- expand32(arr);- final long l = arr[BLOCK_SIZE / 2 - 1];- for (int i = BLOCK_SIZE / 2; i < BLOCK_SIZE; ++i) {- arr[i] += l;- }- }-- // For some reason unrolling seems to help- private static void innerPrefixSum32(long[] arr) {- arr[1] += arr[0];- arr[2] += arr[1];- arr[3] += arr[2];- arr[4] += arr[3];- arr[5] += arr[4];- arr[6] += arr[5];- arr[7] += arr[6];- arr[8] += arr[7];- arr[9] += arr[8];- arr[10] += arr[9];- arr[11] += arr[10];- arr[12] += arr[11];- arr[13] += arr[12];- arr[14] += arr[13];- arr[15] += arr[14];- arr[16] += arr[15];- arr[17] += arr[16];- arr[18] += arr[17];- arr[19] += arr[18];- arr[20] += arr[19];- arr[21] += arr[20];- arr[22] += arr[21];- arr[23] += arr[22];- arr[24] += arr[23];- arr[25] += arr[24];- arr[26] += arr[25];- arr[27] += arr[26];- arr[28] += arr[27];- arr[29] += arr[28];- arr[30] += arr[29];- arr[31] += arr[30];- arr[32] += arr[31];- arr[33] += arr[32];- arr[34] += arr[33];- arr[35] += arr[34];- arr[36] += arr[35];- arr[37] += arr[36];- arr[38] += arr[37];- arr[39] += arr[38];- arr[40] += arr[39];- arr[41] += arr[40];- arr[42] += arr[41];- arr[43] += arr[42];- arr[44] += arr[43];- arr[45] += arr[44];- arr[46] += arr[45];- arr[47] += arr[46];- arr[48] += arr[47];- arr[49] += arr[48];- arr[50] += arr[49];- arr[51] += arr[50];- arr[52] += arr[51];- arr[53] += arr[52];- arr[54] += arr[53];- arr[55] += arr[54];- arr[56] += arr[55];- arr[57] += arr[56];- arr[58] += arr[57];- arr[59] += arr[58];- arr[60] += arr[59];- arr[61] += arr[60];- arr[62] += arr[61];- arr[63] += arr[62];- }-- private static void readLELongs(DataInput in, long[] dst, int offset, int length)- throws IOException {- in.readLongs(dst, offset, length);- for (int i = 0; i < length; ++i) {- dst[offset + i] = Long.reverseBytes(dst[offset + i]);- }- }-- private final long[] tmp = new long[BLOCK_SIZE / 2];-- /** Encode 128 integers from {@code longs} into {@code out}. */- void encode(long[] longs, int bitsPerValue, DataOutput out) throws IOException {- final int nextPrimitive;- final int numLongs;- if (bitsPerValue <= 8) {- nextPrimitive = 8;- numLongs = BLOCK_SIZE / 8;- collapse8(longs);- } else if (bitsPerValue <= 16) {- nextPrimitive = 16;- numLongs = BLOCK_SIZE / 4;- collapse16(longs);- } else {- nextPrimitive = 32;- numLongs = BLOCK_SIZE / 2;- collapse32(longs);- }-- final int numLongsPerShift = bitsPerValue * 2;- int idx = 0;- int shift = nextPrimitive - bitsPerValue;- for (int i = 0; i < numLongsPerShift; ++i) {- tmp[i] = longs[idx++] << shift;- }- for (shift = shift - bitsPerValue; shift >= 0; shift -= bitsPerValue) {- for (int i = 0; i < numLongsPerShift; ++i) {- tmp[i] |= longs[idx++] << shift;- }- }-- final int remainingBitsPerLong = shift + bitsPerValue;- final long maskRemainingBitsPerLong;- if (nextPrimitive == 8) {- maskRemainingBitsPerLong = MASKS8[remainingBitsPerLong];- } else if (nextPrimitive == 16) {- maskRemainingBitsPerLong = MASKS16[remainingBitsPerLong];- } else {- maskRemainingBitsPerLong = MASKS32[remainingBitsPerLong];- }-- int tmpIdx = 0;- int remainingBitsPerValue = bitsPerValue;- while (idx < numLongs) {- if (remainingBitsPerValue >= remainingBitsPerLong) {- remainingBitsPerValue -= remainingBitsPerLong;- tmp[tmpIdx++] |= (longs[idx] >>> remainingBitsPerValue) & maskRemainingBitsPerLong;- if (remainingBitsPerValue == 0) {- idx++;- remainingBitsPerValue = bitsPerValue;- }- } else {- final long mask1, mask2;- if (nextPrimitive == 8) {- mask1 = MASKS8[remainingBitsPerValue];- mask2 = MASKS8[remainingBitsPerLong - remainingBitsPerValue];- } else if (nextPrimitive == 16) {- mask1 = MASKS16[remainingBitsPerValue];- mask2 = MASKS16[remainingBitsPerLong - remainingBitsPerValue];- } else {- mask1 = MASKS32[remainingBitsPerValue];- mask2 = MASKS32[remainingBitsPerLong - remainingBitsPerValue];- }- tmp[tmpIdx] |= (longs[idx++] & mask1) << (remainingBitsPerLong - remainingBitsPerValue);- remainingBitsPerValue = bitsPerValue - remainingBitsPerLong + remainingBitsPerValue;- tmp[tmpIdx++] |= (longs[idx] >>> remainingBitsPerValue) & mask2;- }- }-- for (int i = 0; i < numLongsPerShift; ++i) {- // Java longs are big endian and we want to read little endian longs, so we need to reverse- // bytes- long l = Long.reverseBytes(tmp[i]);- out.writeLong(l);- }- }-- /** Number of bytes required to encode 128 integers of {@code bitsPerValue} bits per value. */- int numBytes(int bitsPerValue) throws IOException {- return bitsPerValue << (BLOCK_SIZE_LOG2 - 3);- }-- private static void decodeSlow(int bitsPerValue, DataInput in, long[] tmp, long[] longs)- throws IOException {- final int numLongs = bitsPerValue << 1;- readLELongs(in, tmp, 0, numLongs);- final long mask = MASKS32[bitsPerValue];- int longsIdx = 0;- int shift = 32 - bitsPerValue;- for (; shift >= 0; shift -= bitsPerValue) {- shiftLongs(tmp, numLongs, longs, longsIdx, shift, mask);- longsIdx += numLongs;- }- final int remainingBitsPerLong = shift + bitsPerValue;- final long mask32RemainingBitsPerLong = MASKS32[remainingBitsPerLong];- int tmpIdx = 0;- int remainingBits = remainingBitsPerLong;- for (; longsIdx < BLOCK_SIZE / 2; ++longsIdx) {- int b = bitsPerValue - remainingBits;- long l = (tmp[tmpIdx++] & MASKS32[remainingBits]) << b;- while (b >= remainingBitsPerLong) {- b -= remainingBitsPerLong;- l |= (tmp[tmpIdx++] & mask32RemainingBitsPerLong) << b;- }- if (b > 0) {- l |= (tmp[tmpIdx] >>> (remainingBitsPerLong - b)) & MASKS32[b];- remainingBits = remainingBitsPerLong - b;- } else {- remainingBits = remainingBitsPerLong;- }- longs[longsIdx] = l;- }- }-- /**- * The pattern that this shiftLongs method applies is recognized by the C2 compiler, which- * generates SIMD instructions for it in order to shift multiple longs at once.- */- private static void shiftLongs(long[] a, int count, long[] b, int bi, int shift, long mask) {- for (int i = 0; i < count; ++i) {- b[bi + i] = (a[i] >>> shift) & mask;- }- }-- private static final long[] MASKS8 = new long[8];- private static final long[] MASKS16 = new long[16];- private static final long[] MASKS32 = new long[32];-- static {- for (int i = 0; i < 8; ++i) {- MASKS8[i] = mask8(i);- }- for (int i = 0; i < 16; ++i) {- MASKS16[i] = mask16(i);- }- for (int i = 0; i < 32; ++i) {- MASKS32[i] = mask32(i);- }- }-- // mark values in array as final longs to avoid the cost of reading array, arrays should only be- // used when the idx is a variable- private static final long MASK8_1 = MASKS8[1];- private static final long MASK8_2 = MASKS8[2];- private static final long MASK8_3 = MASKS8[3];- private static final long MASK8_4 = MASKS8[4];- private static final long MASK8_5 = MASKS8[5];- private static final long MASK8_6 = MASKS8[6];- private static final long MASK8_7 = MASKS8[7];- private static final long MASK16_1 = MASKS16[1];- private static final long MASK16_2 = MASKS16[2];- private static final long MASK16_3 = MASKS16[3];- private static final long MASK16_4 = MASKS16[4];- private static final long MASK16_5 = MASKS16[5];- private static final long MASK16_6 = MASKS16[6];- private static final long MASK16_7 = MASKS16[7];- private static final long MASK16_9 = MASKS16[9];- private static final long MASK16_10 = MASKS16[10];- private static final long MASK16_11 = MASKS16[11];- private static final long MASK16_12 = MASKS16[12];- private static final long MASK16_13 = MASKS16[13];- private static final long MASK16_14 = MASKS16[14];- private static final long MASK16_15 = MASKS16[15];- private static final long MASK32_1 = MASKS32[1];- private static final long MASK32_2 = MASKS32[2];- private static final long MASK32_3 = MASKS32[3];- private static final long MASK32_4 = MASKS32[4];- private static final long MASK32_5 = MASKS32[5];- private static final long MASK32_6 = MASKS32[6];- private static final long MASK32_7 = MASKS32[7];- private static final long MASK32_8 = MASKS32[8];- private static final long MASK32_9 = MASKS32[9];- private static final long MASK32_10 = MASKS32[10];- private static final long MASK32_11 = MASKS32[11];- private static final long MASK32_12 = MASKS32[12];- private static final long MASK32_13 = MASKS32[13];- private static final long MASK32_14 = MASKS32[14];- private static final long MASK32_15 = MASKS32[15];- private static final long MASK32_17 = MASKS32[17];- private static final long MASK32_18 = MASKS32[18];- private static final long MASK32_19 = MASKS32[19];- private static final long MASK32_20 = MASKS32[20];- private static final long MASK32_21 = MASKS32[21];- private static final long MASK32_22 = MASKS32[22];- private static final long MASK32_23 = MASKS32[23];- private static final long MASK32_24 = MASKS32[24];-- /** Decode 128 integers into {@code longs}. */- void decode(int bitsPerValue, DataInput in, long[] longs) throws IOException {- switch (bitsPerValue) {- case 1:- decode1(in, tmp, longs);- expand8(longs);- break;- case 2:- decode2(in, tmp, longs);- expand8(longs);- break;- case 3:- decode3(in, tmp, longs);- expand8(longs);- break;- case 4:- decode4(in, tmp, longs);- expand8(longs);- break;- case 5:- decode5(in, tmp, longs);- expand8(longs);- break;- case 6:- decode6(in, tmp, longs);- expand8(longs);- break;- case 7:- decode7(in, tmp, longs);- expand8(longs);- break;- case 8:- decode8(in, tmp, longs);- expand8(longs);- break;- case 9:- decode9(in, tmp, longs);- expand16(longs);- break;- case 10:- decode10(in, tmp, longs);- expand16(longs);- break;- case 11:- decode11(in, tmp, longs);- expand16(longs);- break;- case 12:- decode12(in, tmp, longs);- expand16(longs);- break;- case 13:- decode13(in, tmp, longs);- expand16(longs);- break;- case 14:- decode14(in, tmp, longs);- expand16(longs);- break;- case 15:- decode15(in, tmp, longs);- expand16(longs);- break;- case 16:- decode16(in, tmp, longs);- expand16(longs);- break;- case 17:- decode17(in, tmp, longs);- expand32(longs);- break;- case 18:- decode18(in, tmp, longs);- expand32(longs);- break;- case 19:- decode19(in, tmp, longs);- expand32(longs);- break;- case 20:- decode20(in, tmp, longs);- expand32(longs);- break;- case 21:- decode21(in, tmp, longs);- expand32(longs);- break;- case 22:- decode22(in, tmp, longs);- expand32(longs);- break;- case 23:- decode23(in, tmp, longs);- expand32(longs);- break;- case 24:- decode24(in, tmp, longs);- expand32(longs);- break;- default:- decodeSlow(bitsPerValue, in, tmp, longs);- expand32(longs);- break;- }- }-- /** Delta-decode 128 integers into {@code longs}. */- void decodeAndPrefixSum(int bitsPerValue, DataInput in, long base, long[] longs)- throws IOException {- switch (bitsPerValue) {- case 1:- decode1(in, tmp, longs);- prefixSum8(longs, base);- break;- case 2:- decode2(in, tmp, longs);- prefixSum8(longs, base);- break;- case 3:- decode3(in, tmp, longs);- prefixSum8(longs, base);- break;- case 4:- decode4(in, tmp, longs);- prefixSum8(longs, base);- break;- case 5:- decode5(in, tmp, longs);- prefixSum8(longs, base);- break;- case 6:- decode6(in, tmp, longs);- prefixSum8(longs, base);- break;- case 7:- decode7(in, tmp, longs);- prefixSum8(longs, base);- break;- case 8:- decode8(in, tmp, longs);- prefixSum8(longs, base);- break;- case 9:- decode9(in, tmp, longs);- prefixSum16(longs, base);- break;- case 10:- decode10(in, tmp, longs);- prefixSum16(longs, base);- break;- case 11:- decode11(in, tmp, longs);- prefixSum16(longs, base);- break;- case 12:- decode12(in, tmp, longs);- prefixSum16(longs, base);- break;- case 13:- decode13(in, tmp, longs);- prefixSum16(longs, base);- break;- case 14:- decode14(in, tmp, longs);- prefixSum16(longs, base);- break;- case 15:- decode15(in, tmp, longs);- prefixSum16(longs, base);- break;- case 16:- decode16(in, tmp, longs);- prefixSum16(longs, base);- break;- case 17:- decode17(in, tmp, longs);- prefixSum32(longs, base);- break;- case 18:- decode18(in, tmp, longs);- prefixSum32(longs, base);- break;- case 19:- decode19(in, tmp, longs);- prefixSum32(longs, base);- break;- case 20:- decode20(in, tmp, longs);- prefixSum32(longs, base);- break;- case 21:- decode21(in, tmp, longs);- prefixSum32(longs, base);- break;- case 22:- decode22(in, tmp, longs);- prefixSum32(longs, base);- break;- case 23:- decode23(in, tmp, longs);- prefixSum32(longs, base);- break;- case 24:- decode24(in, tmp, longs);- prefixSum32(longs, base);- break;- default:- decodeSlow(bitsPerValue, in, tmp, longs);- prefixSum32(longs, base);- break;- }- }-- private static void decode1(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, tmp, 0, 2);- shiftLongs(tmp, 2, longs, 0, 7, MASK8_1);- shiftLongs(tmp, 2, longs, 2, 6, MASK8_1);- shiftLongs(tmp, 2, longs, 4, 5, MASK8_1);- shiftLongs(tmp, 2, longs, 6, 4, MASK8_1);- shiftLongs(tmp, 2, longs, 8, 3, MASK8_1);- shiftLongs(tmp, 2, longs, 10, 2, MASK8_1);- shiftLongs(tmp, 2, longs, 12, 1, MASK8_1);- shiftLongs(tmp, 2, longs, 14, 0, MASK8_1);- }-- private static void decode2(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, tmp, 0, 4);- shiftLongs(tmp, 4, longs, 0, 6, MASK8_2);- shiftLongs(tmp, 4, longs, 4, 4, MASK8_2);- shiftLongs(tmp, 4, longs, 8, 2, MASK8_2);- shiftLongs(tmp, 4, longs, 12, 0, MASK8_2);- }-- private static void decode3(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, tmp, 0, 6);- shiftLongs(tmp, 6, longs, 0, 5, MASK8_3);- shiftLongs(tmp, 6, longs, 6, 2, MASK8_3);- for (int iter = 0, tmpIdx = 0, longsIdx = 12; iter < 2; ++iter, tmpIdx += 3, longsIdx += 2) {- long l0 = (tmp[tmpIdx + 0] & MASK8_2) << 1;- l0 |= (tmp[tmpIdx + 1] >>> 1) & MASK8_1;- longs[longsIdx + 0] = l0;- long l1 = (tmp[tmpIdx + 1] & MASK8_1) << 2;- l1 |= (tmp[tmpIdx + 2] & MASK8_2) << 0;- longs[longsIdx + 1] = l1;- }- }-- private static void decode4(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, tmp, 0, 8);- shiftLongs(tmp, 8, longs, 0, 4, MASK8_4);- shiftLongs(tmp, 8, longs, 8, 0, MASK8_4);- }-- private static void decode5(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, tmp, 0, 10);- shiftLongs(tmp, 10, longs, 0, 3, MASK8_5);- for (int iter = 0, tmpIdx = 0, longsIdx = 10; iter < 2; ++iter, tmpIdx += 5, longsIdx += 3) {- long l0 = (tmp[tmpIdx + 0] & MASK8_3) << 2;- l0 |= (tmp[tmpIdx + 1] >>> 1) & MASK8_2;- longs[longsIdx + 0] = l0;- long l1 = (tmp[tmpIdx + 1] & MASK8_1) << 4;- l1 |= (tmp[tmpIdx + 2] & MASK8_3) << 1;- l1 |= (tmp[tmpIdx + 3] >>> 2) & MASK8_1;- longs[longsIdx + 1] = l1;- long l2 = (tmp[tmpIdx + 3] & MASK8_2) << 3;- l2 |= (tmp[tmpIdx + 4] & MASK8_3) << 0;- longs[longsIdx + 2] = l2;- }- }-- private static void decode6(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, tmp, 0, 12);- shiftLongs(tmp, 12, longs, 0, 2, MASK8_6);- shiftLongs(tmp, 12, tmp, 0, 0, MASK8_2);- for (int iter = 0, tmpIdx = 0, longsIdx = 12; iter < 4; ++iter, tmpIdx += 3, longsIdx += 1) {- long l0 = tmp[tmpIdx + 0] << 4;- l0 |= tmp[tmpIdx + 1] << 2;- l0 |= tmp[tmpIdx + 2] << 0;- longs[longsIdx + 0] = l0;- }- }-- private static void decode7(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, tmp, 0, 14);- shiftLongs(tmp, 14, longs, 0, 1, MASK8_7);- shiftLongs(tmp, 14, tmp, 0, 0, MASK8_1);- for (int iter = 0, tmpIdx = 0, longsIdx = 14; iter < 2; ++iter, tmpIdx += 7, longsIdx += 1) {- long l0 = tmp[tmpIdx + 0] << 6;- l0 |= tmp[tmpIdx + 1] << 5;- l0 |= tmp[tmpIdx + 2] << 4;- l0 |= tmp[tmpIdx + 3] << 3;- l0 |= tmp[tmpIdx + 4] << 2;- l0 |= tmp[tmpIdx + 5] << 1;- l0 |= tmp[tmpIdx + 6] << 0;- longs[longsIdx + 0] = l0;- }- }-- private static void decode8(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, longs, 0, 16);- }-- private static void decode9(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, tmp, 0, 18);- shiftLongs(tmp, 18, longs, 0, 7, MASK16_9);- for (int iter = 0, tmpIdx = 0, longsIdx = 18; iter < 2; ++iter, tmpIdx += 9, longsIdx += 7) {- long l0 = (tmp[tmpIdx + 0] & MASK16_7) << 2;- l0 |= (tmp[tmpIdx + 1] >>> 5) & MASK16_2;- longs[longsIdx + 0] = l0;- long l1 = (tmp[tmpIdx + 1] & MASK16_5) << 4;- l1 |= (tmp[tmpIdx + 2] >>> 3) & MASK16_4;- longs[longsIdx + 1] = l1;- long l2 = (tmp[tmpIdx + 2] & MASK16_3) << 6;- l2 |= (tmp[tmpIdx + 3] >>> 1) & MASK16_6;- longs[longsIdx + 2] = l2;- long l3 = (tmp[tmpIdx + 3] & MASK16_1) << 8;- l3 |= (tmp[tmpIdx + 4] & MASK16_7) << 1;- l3 |= (tmp[tmpIdx + 5] >>> 6) & MASK16_1;- longs[longsIdx + 3] = l3;- long l4 = (tmp[tmpIdx + 5] & MASK16_6) << 3;- l4 |= (tmp[tmpIdx + 6] >>> 4) & MASK16_3;- longs[longsIdx + 4] = l4;- long l5 = (tmp[tmpIdx + 6] & MASK16_4) << 5;- l5 |= (tmp[tmpIdx + 7] >>> 2) & MASK16_5;- longs[longsIdx + 5] = l5;- long l6 = (tmp[tmpIdx + 7] & MASK16_2) << 7;- l6 |= (tmp[tmpIdx + 8] & MASK16_7) << 0;- longs[longsIdx + 6] = l6;- }- }-- private static void decode10(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, tmp, 0, 20);- shiftLongs(tmp, 20, longs, 0, 6, MASK16_10);- for (int iter = 0, tmpIdx = 0, longsIdx = 20; iter < 4; ++iter, tmpIdx += 5, longsIdx += 3) {- long l0 = (tmp[tmpIdx + 0] & MASK16_6) << 4;- l0 |= (tmp[tmpIdx + 1] >>> 2) & MASK16_4;- longs[longsIdx + 0] = l0;- long l1 = (tmp[tmpIdx + 1] & MASK16_2) << 8;- l1 |= (tmp[tmpIdx + 2] & MASK16_6) << 2;- l1 |= (tmp[tmpIdx + 3] >>> 4) & MASK16_2;- longs[longsIdx + 1] = l1;- long l2 = (tmp[tmpIdx + 3] & MASK16_4) << 6;- l2 |= (tmp[tmpIdx + 4] & MASK16_6) << 0;- longs[longsIdx + 2] = l2;- }- }-- private static void decode11(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, tmp, 0, 22);- shiftLongs(tmp, 22, longs, 0, 5, MASK16_11);- for (int iter = 0, tmpIdx = 0, longsIdx = 22; iter < 2; ++iter, tmpIdx += 11, longsIdx += 5) {- long l0 = (tmp[tmpIdx + 0] & MASK16_5) << 6;- l0 |= (tmp[tmpIdx + 1] & MASK16_5) << 1;- l0 |= (tmp[tmpIdx + 2] >>> 4) & MASK16_1;- longs[longsIdx + 0] = l0;- long l1 = (tmp[tmpIdx + 2] & MASK16_4) << 7;- l1 |= (tmp[tmpIdx + 3] & MASK16_5) << 2;- l1 |= (tmp[tmpIdx + 4] >>> 3) & MASK16_2;- longs[longsIdx + 1] = l1;- long l2 = (tmp[tmpIdx + 4] & MASK16_3) << 8;- l2 |= (tmp[tmpIdx + 5] & MASK16_5) << 3;- l2 |= (tmp[tmpIdx + 6] >>> 2) & MASK16_3;- longs[longsIdx + 2] = l2;- long l3 = (tmp[tmpIdx + 6] & MASK16_2) << 9;- l3 |= (tmp[tmpIdx + 7] & MASK16_5) << 4;- l3 |= (tmp[tmpIdx + 8] >>> 1) & MASK16_4;- longs[longsIdx + 3] = l3;- long l4 = (tmp[tmpIdx + 8] & MASK16_1) << 10;- l4 |= (tmp[tmpIdx + 9] & MASK16_5) << 5;- l4 |= (tmp[tmpIdx + 10] & MASK16_5) << 0;- longs[longsIdx + 4] = l4;- }- }-- private static void decode12(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, tmp, 0, 24);- shiftLongs(tmp, 24, longs, 0, 4, MASK16_12);- shiftLongs(tmp, 24, tmp, 0, 0, MASK16_4);- for (int iter = 0, tmpIdx = 0, longsIdx = 24; iter < 8; ++iter, tmpIdx += 3, longsIdx += 1) {- long l0 = tmp[tmpIdx + 0] << 8;- l0 |= tmp[tmpIdx + 1] << 4;- l0 |= tmp[tmpIdx + 2] << 0;- longs[longsIdx + 0] = l0;- }- }-- private static void decode13(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, tmp, 0, 26);- shiftLongs(tmp, 26, longs, 0, 3, MASK16_13);- for (int iter = 0, tmpIdx = 0, longsIdx = 26; iter < 2; ++iter, tmpIdx += 13, longsIdx += 3) {- long l0 = (tmp[tmpIdx + 0] & MASK16_3) << 10;- l0 |= (tmp[tmpIdx + 1] & MASK16_3) << 7;- l0 |= (tmp[tmpIdx + 2] & MASK16_3) << 4;- l0 |= (tmp[tmpIdx + 3] & MASK16_3) << 1;- l0 |= (tmp[tmpIdx + 4] >>> 2) & MASK16_1;- longs[longsIdx + 0] = l0;- long l1 = (tmp[tmpIdx + 4] & MASK16_2) << 11;- l1 |= (tmp[tmpIdx + 5] & MASK16_3) << 8;- l1 |= (tmp[tmpIdx + 6] & MASK16_3) << 5;- l1 |= (tmp[tmpIdx + 7] & MASK16_3) << 2;- l1 |= (tmp[tmpIdx + 8] >>> 1) & MASK16_2;- longs[longsIdx + 1] = l1;- long l2 = (tmp[tmpIdx + 8] & MASK16_1) << 12;- l2 |= (tmp[tmpIdx + 9] & MASK16_3) << 9;- l2 |= (tmp[tmpIdx + 10] & MASK16_3) << 6;- l2 |= (tmp[tmpIdx + 11] & MASK16_3) << 3;- l2 |= (tmp[tmpIdx + 12] & MASK16_3) << 0;- longs[longsIdx + 2] = l2;- }- }-- private static void decode14(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, tmp, 0, 28);- shiftLongs(tmp, 28, longs, 0, 2, MASK16_14);- shiftLongs(tmp, 28, tmp, 0, 0, MASK16_2);- for (int iter = 0, tmpIdx = 0, longsIdx = 28; iter < 4; ++iter, tmpIdx += 7, longsIdx += 1) {- long l0 = tmp[tmpIdx + 0] << 12;- l0 |= tmp[tmpIdx + 1] << 10;- l0 |= tmp[tmpIdx + 2] << 8;- l0 |= tmp[tmpIdx + 3] << 6;- l0 |= tmp[tmpIdx + 4] << 4;- l0 |= tmp[tmpIdx + 5] << 2;- l0 |= tmp[tmpIdx + 6] << 0;- longs[longsIdx + 0] = l0;- }- }-- private static void decode15(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, tmp, 0, 30);- shiftLongs(tmp, 30, longs, 0, 1, MASK16_15);- shiftLongs(tmp, 30, tmp, 0, 0, MASK16_1);- for (int iter = 0, tmpIdx = 0, longsIdx = 30; iter < 2; ++iter, tmpIdx += 15, longsIdx += 1) {- long l0 = tmp[tmpIdx + 0] << 14;- l0 |= tmp[tmpIdx + 1] << 13;- l0 |= tmp[tmpIdx + 2] << 12;- l0 |= tmp[tmpIdx + 3] << 11;- l0 |= tmp[tmpIdx + 4] << 10;- l0 |= tmp[tmpIdx + 5] << 9;- l0 |= tmp[tmpIdx + 6] << 8;- l0 |= tmp[tmpIdx + 7] << 7;- l0 |= tmp[tmpIdx + 8] << 6;- l0 |= tmp[tmpIdx + 9] << 5;- l0 |= tmp[tmpIdx + 10] << 4;- l0 |= tmp[tmpIdx + 11] << 3;- l0 |= tmp[tmpIdx + 12] << 2;- l0 |= tmp[tmpIdx + 13] << 1;- l0 |= tmp[tmpIdx + 14] << 0;- longs[longsIdx + 0] = l0;- }- }-- private static void decode16(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, longs, 0, 32);- }-- private static void decode17(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, tmp, 0, 34);- shiftLongs(tmp, 34, longs, 0, 15, MASK32_17);- for (int iter = 0, tmpIdx = 0, longsIdx = 34; iter < 2; ++iter, tmpIdx += 17, longsIdx += 15) {- long l0 = (tmp[tmpIdx + 0] & MASK32_15) << 2;- l0 |= (tmp[tmpIdx + 1] >>> 13) & MASK32_2;- longs[longsIdx + 0] = l0;- long l1 = (tmp[tmpIdx + 1] & MASK32_13) << 4;- l1 |= (tmp[tmpIdx + 2] >>> 11) & MASK32_4;- longs[longsIdx + 1] = l1;- long l2 = (tmp[tmpIdx + 2] & MASK32_11) << 6;- l2 |= (tmp[tmpIdx + 3] >>> 9) & MASK32_6;- longs[longsIdx + 2] = l2;- long l3 = (tmp[tmpIdx + 3] & MASK32_9) << 8;- l3 |= (tmp[tmpIdx + 4] >>> 7) & MASK32_8;- longs[longsIdx + 3] = l3;- long l4 = (tmp[tmpIdx + 4] & MASK32_7) << 10;- l4 |= (tmp[tmpIdx + 5] >>> 5) & MASK32_10;- longs[longsIdx + 4] = l4;- long l5 = (tmp[tmpIdx + 5] & MASK32_5) << 12;- l5 |= (tmp[tmpIdx + 6] >>> 3) & MASK32_12;- longs[longsIdx + 5] = l5;- long l6 = (tmp[tmpIdx + 6] & MASK32_3) << 14;- l6 |= (tmp[tmpIdx + 7] >>> 1) & MASK32_14;- longs[longsIdx + 6] = l6;- long l7 = (tmp[tmpIdx + 7] & MASK32_1) << 16;- l7 |= (tmp[tmpIdx + 8] & MASK32_15) << 1;- l7 |= (tmp[tmpIdx + 9] >>> 14) & MASK32_1;- longs[longsIdx + 7] = l7;- long l8 = (tmp[tmpIdx + 9] & MASK32_14) << 3;- l8 |= (tmp[tmpIdx + 10] >>> 12) & MASK32_3;- longs[longsIdx + 8] = l8;- long l9 = (tmp[tmpIdx + 10] & MASK32_12) << 5;- l9 |= (tmp[tmpIdx + 11] >>> 10) & MASK32_5;- longs[longsIdx + 9] = l9;- long l10 = (tmp[tmpIdx + 11] & MASK32_10) << 7;- l10 |= (tmp[tmpIdx + 12] >>> 8) & MASK32_7;- longs[longsIdx + 10] = l10;- long l11 = (tmp[tmpIdx + 12] & MASK32_8) << 9;- l11 |= (tmp[tmpIdx + 13] >>> 6) & MASK32_9;- longs[longsIdx + 11] = l11;- long l12 = (tmp[tmpIdx + 13] & MASK32_6) << 11;- l12 |= (tmp[tmpIdx + 14] >>> 4) & MASK32_11;- longs[longsIdx + 12] = l12;- long l13 = (tmp[tmpIdx + 14] & MASK32_4) << 13;- l13 |= (tmp[tmpIdx + 15] >>> 2) & MASK32_13;- longs[longsIdx + 13] = l13;- long l14 = (tmp[tmpIdx + 15] & MASK32_2) << 15;- l14 |= (tmp[tmpIdx + 16] & MASK32_15) << 0;- longs[longsIdx + 14] = l14;- }- }-- private static void decode18(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, tmp, 0, 36);- shiftLongs(tmp, 36, longs, 0, 14, MASK32_18);- for (int iter = 0, tmpIdx = 0, longsIdx = 36; iter < 4; ++iter, tmpIdx += 9, longsIdx += 7) {- long l0 = (tmp[tmpIdx + 0] & MASK32_14) << 4;- l0 |= (tmp[tmpIdx + 1] >>> 10) & MASK32_4;- longs[longsIdx + 0] = l0;- long l1 = (tmp[tmpIdx + 1] & MASK32_10) << 8;- l1 |= (tmp[tmpIdx + 2] >>> 6) & MASK32_8;- longs[longsIdx + 1] = l1;- long l2 = (tmp[tmpIdx + 2] & MASK32_6) << 12;- l2 |= (tmp[tmpIdx + 3] >>> 2) & MASK32_12;- longs[longsIdx + 2] = l2;- long l3 = (tmp[tmpIdx + 3] & MASK32_2) << 16;- l3 |= (tmp[tmpIdx + 4] & MASK32_14) << 2;- l3 |= (tmp[tmpIdx + 5] >>> 12) & MASK32_2;- longs[longsIdx + 3] = l3;- long l4 = (tmp[tmpIdx + 5] & MASK32_12) << 6;- l4 |= (tmp[tmpIdx + 6] >>> 8) & MASK32_6;- longs[longsIdx + 4] = l4;- long l5 = (tmp[tmpIdx + 6] & MASK32_8) << 10;- l5 |= (tmp[tmpIdx + 7] >>> 4) & MASK32_10;- longs[longsIdx + 5] = l5;- long l6 = (tmp[tmpIdx + 7] & MASK32_4) << 14;- l6 |= (tmp[tmpIdx + 8] & MASK32_14) << 0;- longs[longsIdx + 6] = l6;- }- }-- private static void decode19(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, tmp, 0, 38);- shiftLongs(tmp, 38, longs, 0, 13, MASK32_19);- for (int iter = 0, tmpIdx = 0, longsIdx = 38; iter < 2; ++iter, tmpIdx += 19, longsIdx += 13) {- long l0 = (tmp[tmpIdx + 0] & MASK32_13) << 6;- l0 |= (tmp[tmpIdx + 1] >>> 7) & MASK32_6;- longs[longsIdx + 0] = l0;- long l1 = (tmp[tmpIdx + 1] & MASK32_7) << 12;- l1 |= (tmp[tmpIdx + 2] >>> 1) & MASK32_12;- longs[longsIdx + 1] = l1;- long l2 = (tmp[tmpIdx + 2] & MASK32_1) << 18;- l2 |= (tmp[tmpIdx + 3] & MASK32_13) << 5;- l2 |= (tmp[tmpIdx + 4] >>> 8) & MASK32_5;- longs[longsIdx + 2] = l2;- long l3 = (tmp[tmpIdx + 4] & MASK32_8) << 11;- l3 |= (tmp[tmpIdx + 5] >>> 2) & MASK32_11;- longs[longsIdx + 3] = l3;- long l4 = (tmp[tmpIdx + 5] & MASK32_2) << 17;- l4 |= (tmp[tmpIdx + 6] & MASK32_13) << 4;- l4 |= (tmp[tmpIdx + 7] >>> 9) & MASK32_4;- longs[longsIdx + 4] = l4;- long l5 = (tmp[tmpIdx + 7] & MASK32_9) << 10;- l5 |= (tmp[tmpIdx + 8] >>> 3) & MASK32_10;- longs[longsIdx + 5] = l5;- long l6 = (tmp[tmpIdx + 8] & MASK32_3) << 16;- l6 |= (tmp[tmpIdx + 9] & MASK32_13) << 3;- l6 |= (tmp[tmpIdx + 10] >>> 10) & MASK32_3;- longs[longsIdx + 6] = l6;- long l7 = (tmp[tmpIdx + 10] & MASK32_10) << 9;- l7 |= (tmp[tmpIdx + 11] >>> 4) & MASK32_9;- longs[longsIdx + 7] = l7;- long l8 = (tmp[tmpIdx + 11] & MASK32_4) << 15;- l8 |= (tmp[tmpIdx + 12] & MASK32_13) << 2;- l8 |= (tmp[tmpIdx + 13] >>> 11) & MASK32_2;- longs[longsIdx + 8] = l8;- long l9 = (tmp[tmpIdx + 13] & MASK32_11) << 8;- l9 |= (tmp[tmpIdx + 14] >>> 5) & MASK32_8;- longs[longsIdx + 9] = l9;- long l10 = (tmp[tmpIdx + 14] & MASK32_5) << 14;- l10 |= (tmp[tmpIdx + 15] & MASK32_13) << 1;- l10 |= (tmp[tmpIdx + 16] >>> 12) & MASK32_1;- longs[longsIdx + 10] = l10;- long l11 = (tmp[tmpIdx + 16] & MASK32_12) << 7;- l11 |= (tmp[tmpIdx + 17] >>> 6) & MASK32_7;- longs[longsIdx + 11] = l11;- long l12 = (tmp[tmpIdx + 17] & MASK32_6) << 13;- l12 |= (tmp[tmpIdx + 18] & MASK32_13) << 0;- longs[longsIdx + 12] = l12;- }- }-- private static void decode20(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, tmp, 0, 40);- shiftLongs(tmp, 40, longs, 0, 12, MASK32_20);- for (int iter = 0, tmpIdx = 0, longsIdx = 40; iter < 8; ++iter, tmpIdx += 5, longsIdx += 3) {- long l0 = (tmp[tmpIdx + 0] & MASK32_12) << 8;- l0 |= (tmp[tmpIdx + 1] >>> 4) & MASK32_8;- longs[longsIdx + 0] = l0;- long l1 = (tmp[tmpIdx + 1] & MASK32_4) << 16;- l1 |= (tmp[tmpIdx + 2] & MASK32_12) << 4;- l1 |= (tmp[tmpIdx + 3] >>> 8) & MASK32_4;- longs[longsIdx + 1] = l1;- long l2 = (tmp[tmpIdx + 3] & MASK32_8) << 12;- l2 |= (tmp[tmpIdx + 4] & MASK32_12) << 0;- longs[longsIdx + 2] = l2;- }- }-- private static void decode21(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, tmp, 0, 42);- shiftLongs(tmp, 42, longs, 0, 11, MASK32_21);- for (int iter = 0, tmpIdx = 0, longsIdx = 42; iter < 2; ++iter, tmpIdx += 21, longsIdx += 11) {- long l0 = (tmp[tmpIdx + 0] & MASK32_11) << 10;- l0 |= (tmp[tmpIdx + 1] >>> 1) & MASK32_10;- longs[longsIdx + 0] = l0;- long l1 = (tmp[tmpIdx + 1] & MASK32_1) << 20;- l1 |= (tmp[tmpIdx + 2] & MASK32_11) << 9;- l1 |= (tmp[tmpIdx + 3] >>> 2) & MASK32_9;- longs[longsIdx + 1] = l1;- long l2 = (tmp[tmpIdx + 3] & MASK32_2) << 19;- l2 |= (tmp[tmpIdx + 4] & MASK32_11) << 8;- l2 |= (tmp[tmpIdx + 5] >>> 3) & MASK32_8;- longs[longsIdx + 2] = l2;- long l3 = (tmp[tmpIdx + 5] & MASK32_3) << 18;- l3 |= (tmp[tmpIdx + 6] & MASK32_11) << 7;- l3 |= (tmp[tmpIdx + 7] >>> 4) & MASK32_7;- longs[longsIdx + 3] = l3;- long l4 = (tmp[tmpIdx + 7] & MASK32_4) << 17;- l4 |= (tmp[tmpIdx + 8] & MASK32_11) << 6;- l4 |= (tmp[tmpIdx + 9] >>> 5) & MASK32_6;- longs[longsIdx + 4] = l4;- long l5 = (tmp[tmpIdx + 9] & MASK32_5) << 16;- l5 |= (tmp[tmpIdx + 10] & MASK32_11) << 5;- l5 |= (tmp[tmpIdx + 11] >>> 6) & MASK32_5;- longs[longsIdx + 5] = l5;- long l6 = (tmp[tmpIdx + 11] & MASK32_6) << 15;- l6 |= (tmp[tmpIdx + 12] & MASK32_11) << 4;- l6 |= (tmp[tmpIdx + 13] >>> 7) & MASK32_4;- longs[longsIdx + 6] = l6;- long l7 = (tmp[tmpIdx + 13] & MASK32_7) << 14;- l7 |= (tmp[tmpIdx + 14] & MASK32_11) << 3;- l7 |= (tmp[tmpIdx + 15] >>> 8) & MASK32_3;- longs[longsIdx + 7] = l7;- long l8 = (tmp[tmpIdx + 15] & MASK32_8) << 13;- l8 |= (tmp[tmpIdx + 16] & MASK32_11) << 2;- l8 |= (tmp[tmpIdx + 17] >>> 9) & MASK32_2;- longs[longsIdx + 8] = l8;- long l9 = (tmp[tmpIdx + 17] & MASK32_9) << 12;- l9 |= (tmp[tmpIdx + 18] & MASK32_11) << 1;- l9 |= (tmp[tmpIdx + 19] >>> 10) & MASK32_1;- longs[longsIdx + 9] = l9;- long l10 = (tmp[tmpIdx + 19] & MASK32_10) << 11;- l10 |= (tmp[tmpIdx + 20] & MASK32_11) << 0;- longs[longsIdx + 10] = l10;- }- }-- private static void decode22(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, tmp, 0, 44);- shiftLongs(tmp, 44, longs, 0, 10, MASK32_22);- for (int iter = 0, tmpIdx = 0, longsIdx = 44; iter < 4; ++iter, tmpIdx += 11, longsIdx += 5) {- long l0 = (tmp[tmpIdx + 0] & MASK32_10) << 12;- l0 |= (tmp[tmpIdx + 1] & MASK32_10) << 2;- l0 |= (tmp[tmpIdx + 2] >>> 8) & MASK32_2;- longs[longsIdx + 0] = l0;- long l1 = (tmp[tmpIdx + 2] & MASK32_8) << 14;- l1 |= (tmp[tmpIdx + 3] & MASK32_10) << 4;- l1 |= (tmp[tmpIdx + 4] >>> 6) & MASK32_4;- longs[longsIdx + 1] = l1;- long l2 = (tmp[tmpIdx + 4] & MASK32_6) << 16;- l2 |= (tmp[tmpIdx + 5] & MASK32_10) << 6;- l2 |= (tmp[tmpIdx + 6] >>> 4) & MASK32_6;- longs[longsIdx + 2] = l2;- long l3 = (tmp[tmpIdx + 6] & MASK32_4) << 18;- l3 |= (tmp[tmpIdx + 7] & MASK32_10) << 8;- l3 |= (tmp[tmpIdx + 8] >>> 2) & MASK32_8;- longs[longsIdx + 3] = l3;- long l4 = (tmp[tmpIdx + 8] & MASK32_2) << 20;- l4 |= (tmp[tmpIdx + 9] & MASK32_10) << 10;- l4 |= (tmp[tmpIdx + 10] & MASK32_10) << 0;- longs[longsIdx + 4] = l4;- }- }-- private static void decode23(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, tmp, 0, 46);- shiftLongs(tmp, 46, longs, 0, 9, MASK32_23);- for (int iter = 0, tmpIdx = 0, longsIdx = 46; iter < 2; ++iter, tmpIdx += 23, longsIdx += 9) {- long l0 = (tmp[tmpIdx + 0] & MASK32_9) << 14;- l0 |= (tmp[tmpIdx + 1] & MASK32_9) << 5;- l0 |= (tmp[tmpIdx + 2] >>> 4) & MASK32_5;- longs[longsIdx + 0] = l0;- long l1 = (tmp[tmpIdx + 2] & MASK32_4) << 19;- l1 |= (tmp[tmpIdx + 3] & MASK32_9) << 10;- l1 |= (tmp[tmpIdx + 4] & MASK32_9) << 1;- l1 |= (tmp[tmpIdx + 5] >>> 8) & MASK32_1;- longs[longsIdx + 1] = l1;- long l2 = (tmp[tmpIdx + 5] & MASK32_8) << 15;- l2 |= (tmp[tmpIdx + 6] & MASK32_9) << 6;- l2 |= (tmp[tmpIdx + 7] >>> 3) & MASK32_6;- longs[longsIdx + 2] = l2;- long l3 = (tmp[tmpIdx + 7] & MASK32_3) << 20;- l3 |= (tmp[tmpIdx + 8] & MASK32_9) << 11;- l3 |= (tmp[tmpIdx + 9] & MASK32_9) << 2;- l3 |= (tmp[tmpIdx + 10] >>> 7) & MASK32_2;- longs[longsIdx + 3] = l3;- long l4 = (tmp[tmpIdx + 10] & MASK32_7) << 16;- l4 |= (tmp[tmpIdx + 11] & MASK32_9) << 7;- l4 |= (tmp[tmpIdx + 12] >>> 2) & MASK32_7;- longs[longsIdx + 4] = l4;- long l5 = (tmp[tmpIdx + 12] & MASK32_2) << 21;- l5 |= (tmp[tmpIdx + 13] & MASK32_9) << 12;- l5 |= (tmp[tmpIdx + 14] & MASK32_9) << 3;- l5 |= (tmp[tmpIdx + 15] >>> 6) & MASK32_3;- longs[longsIdx + 5] = l5;- long l6 = (tmp[tmpIdx + 15] & MASK32_6) << 17;- l6 |= (tmp[tmpIdx + 16] & MASK32_9) << 8;- l6 |= (tmp[tmpIdx + 17] >>> 1) & MASK32_8;- longs[longsIdx + 6] = l6;- long l7 = (tmp[tmpIdx + 17] & MASK32_1) << 22;- l7 |= (tmp[tmpIdx + 18] & MASK32_9) << 13;- l7 |= (tmp[tmpIdx + 19] & MASK32_9) << 4;- l7 |= (tmp[tmpIdx + 20] >>> 5) & MASK32_4;- longs[longsIdx + 7] = l7;- long l8 = (tmp[tmpIdx + 20] & MASK32_5) << 18;- l8 |= (tmp[tmpIdx + 21] & MASK32_9) << 9;- l8 |= (tmp[tmpIdx + 22] & MASK32_9) << 0;- longs[longsIdx + 8] = l8;- }- }-- private static void decode24(DataInput in, long[] tmp, long[] longs) throws IOException {- readLELongs(in, tmp, 0, 48);- shiftLongs(tmp, 48, longs, 0, 8, MASK32_24);- shiftLongs(tmp, 48, tmp, 0, 0, MASK32_8);- for (int iter = 0, tmpIdx = 0, longsIdx = 48; iter < 16; ++iter, tmpIdx += 3, longsIdx += 1) {- long l0 = tmp[tmpIdx + 0] << 16;- l0 |= tmp[tmpIdx + 1] << 8;- l0 |= tmp[tmpIdx + 2] << 0;- longs[longsIdx + 0] = l0;- }- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84Codec.javadeleted file mode 100644index e2f45ca5961..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84Codec.java+++ /dev/null@@ -1,179 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene84;--import java.util.Objects;-import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;-import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;-import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat;-import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat.Mode;-import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat;-import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;-import org.apache.lucene.backward_codecs.lucene60.Lucene60PointsFormat;-import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat;-import org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat;-import org.apache.lucene.codecs.Codec;-import org.apache.lucene.codecs.CompoundFormat;-import org.apache.lucene.codecs.DocValuesFormat;-import org.apache.lucene.codecs.FieldInfosFormat;-import org.apache.lucene.codecs.FilterCodec;-import org.apache.lucene.codecs.KnnVectorsFormat;-import org.apache.lucene.codecs.LiveDocsFormat;-import org.apache.lucene.codecs.NormsFormat;-import org.apache.lucene.codecs.PointsFormat;-import org.apache.lucene.codecs.PostingsFormat;-import org.apache.lucene.codecs.SegmentInfoFormat;-import org.apache.lucene.codecs.StoredFieldsFormat;-import org.apache.lucene.codecs.TermVectorsFormat;-import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;-import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;--/**- * Implements the Lucene 8.4 index format, with configurable per-field postings and docvalues- * formats.- *- * <p>If you want to reuse functionality of this codec in another codec, extend {@link FilterCodec}.- *- * @see org.apache.lucene.backward_codecs.lucene84 package documentation for file format details.- * @lucene.experimental- */-public class Lucene84Codec extends Codec {- private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat();- private final FieldInfosFormat fieldInfosFormat = new Lucene60FieldInfosFormat();- private final SegmentInfoFormat segmentInfosFormat = new Lucene70SegmentInfoFormat();- private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();- private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();- private final PostingsFormat defaultFormat;-- private final PostingsFormat postingsFormat =- new PerFieldPostingsFormat() {- @Override- public PostingsFormat getPostingsFormatForField(String field) {- return Lucene84Codec.this.getPostingsFormatForField(field);- }- };-- private final DocValuesFormat docValuesFormat =- new PerFieldDocValuesFormat() {- @Override- public DocValuesFormat getDocValuesFormatForField(String field) {- return Lucene84Codec.this.getDocValuesFormatForField(field);- }- };-- private final StoredFieldsFormat storedFieldsFormat;-- /** Instantiates a new codec. */- public Lucene84Codec() {- this(Mode.BEST_SPEED);- }-- /**- * Instantiates a new codec, specifying the stored fields compression mode to use.- *- * @param mode stored fields compression mode to use for newly flushed/merged segments.- */- public Lucene84Codec(Mode mode) {- super("Lucene84");- this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Objects.requireNonNull(mode));- this.defaultFormat = new Lucene84PostingsFormat();- }-- @Override- public StoredFieldsFormat storedFieldsFormat() {- return storedFieldsFormat;- }-- @Override- public TermVectorsFormat termVectorsFormat() {- return vectorsFormat;- }-- @Override- public PostingsFormat postingsFormat() {- return postingsFormat;- }-- @Override- public final FieldInfosFormat fieldInfosFormat() {- return fieldInfosFormat;- }-- @Override- public SegmentInfoFormat segmentInfoFormat() {- return segmentInfosFormat;- }-- @Override- public final LiveDocsFormat liveDocsFormat() {- return liveDocsFormat;- }-- @Override- public CompoundFormat compoundFormat() {- return compoundFormat;- }-- @Override- public PointsFormat pointsFormat() {- return new Lucene60PointsFormat();- }-- @Override- public KnnVectorsFormat knnVectorsFormat() {- return KnnVectorsFormat.EMPTY;- }-- /**- * Returns the postings format that should be used for writing new segments of <code>field</code>.- *- * <p>The default implementation always returns "Lucene84".- *- * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:- * future version of Lucene are only guaranteed to be able to read the default implementation.- */- public PostingsFormat getPostingsFormatForField(String field) {- return defaultFormat;- }-- /**- * Returns the docvalues format that should be used for writing new segments of <code>field</code>- * .- *- * <p>The default implementation always returns "Lucene80".- *- * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:- * future version of Lucene are only guaranteed to be able to read the default implementation.- */- public DocValuesFormat getDocValuesFormatForField(String field) {- return defaultDVFormat;- }-- @Override- public final DocValuesFormat docValuesFormat() {- return docValuesFormat;- }-- private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene80");-- private final NormsFormat normsFormat = new Lucene80NormsFormat();-- @Override- public NormsFormat normsFormat() {- return normsFormat;- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84PostingsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84PostingsFormat.javadeleted file mode 100644index 0c63d244fbc..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84PostingsFormat.java+++ /dev/null@@ -1,474 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene84;--import java.io.IOException;-import org.apache.lucene.backward_codecs.lucene40.blocktree.Lucene40BlockTreeTermsReader;-import org.apache.lucene.codecs.BlockTermState;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.FieldsConsumer;-import org.apache.lucene.codecs.FieldsProducer;-import org.apache.lucene.codecs.MultiLevelSkipListWriter;-import org.apache.lucene.codecs.PostingsFormat;-import org.apache.lucene.codecs.PostingsReaderBase;-import org.apache.lucene.index.IndexOptions;-import org.apache.lucene.index.SegmentReadState;-import org.apache.lucene.index.SegmentWriteState;-import org.apache.lucene.index.TermState;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.util.IOUtils;-import org.apache.lucene.util.packed.PackedInts;--/**- * Lucene 5.0 postings format, which encodes postings in packed integer blocks for fast decode.- *- * <p>Basic idea:- *- * <ul>- * <li><b>Packed Blocks and VInt Blocks</b>:- * <p>In packed blocks, integers are encoded with the same bit width ({@link PackedInts packed- * format}): the block size (i.e. number of integers inside block) is fixed (currently 128).- * Additionally blocks that are all the same value are encoded in an optimized way.- * <p>In VInt blocks, integers are encoded as {@link DataOutput#writeVInt VInt}: the block- * size is variable.- * <li><b>Block structure</b>:- * <p>When the postings are long enough, Lucene84PostingsFormat will try to encode most- * integer data as a packed block.- * <p>Take a term with 259 documents as an example, the first 256 document ids are encoded as- * two packed blocks, while the remaining 3 are encoded as one VInt block.- * <p>Different kinds of data are always encoded separately into different packed blocks, but- * may possibly be interleaved into the same VInt block.- * <p>This strategy is applied to pairs: <document number, frequency>, <position,- * payload length>, <position, offset start, offset length>, and <position,- * payload length, offsetstart, offset length>.- * <li><b>Skipdata settings</b>:- * <p>The structure of skip table is quite similar to previous version of Lucene. Skip- * interval is the same as block size, and each skip entry points to the beginning of each- * block. However, for the first block, skip data is omitted.- * <li><b>Positions, Payloads, and Offsets</b>:- * <p>A position is an integer indicating where the term occurs within one document. A payload- * is a blob of metadata associated with current position. An offset is a pair of integers- * indicating the tokenized start/end offsets for given term in current position: it is- * essentially a specialized payload.- * <p>When payloads and offsets are not omitted, numPositions==numPayloads==numOffsets- * (assuming a null payload contributes one count). As mentioned in block structure, it is- * possible to encode these three either combined or separately.- * <p>In all cases, payloads and offsets are stored together. When encoded as a packed block,- * position data is separated out as .pos, while payloads and offsets are encoded in .pay- * (payload metadata will also be stored directly in .pay). When encoded as VInt blocks, all- * these three are stored interleaved into the .pos (so is payload metadata).- * <p>With this strategy, the majority of payload and offset data will be outside .pos file.- * So for queries that require only position data, running on a full index with payloads and- * offsets, this reduces disk pre-fetches.- * </ul>- *- * <p>Files and detailed format:- *- * <ul>- * <li><code>.tim</code>: <a href="#Termdictionary">Term Dictionary</a>- * <li><code>.tip</code>: <a href="#Termindex">Term Index</a>- * <li><code>.doc</code>: <a href="#Frequencies">Frequencies and Skip Data</a>- * <li><code>.pos</code>: <a href="#Positions">Positions</a>- * <li><code>.pay</code>: <a href="#Payloads">Payloads and Offsets</a>- * </ul>- *- * <a id="Termdictionary"></a>- *- * <dl>- * <dd><b>Term Dictionary</b>- * <p>The .tim file contains the list of terms in each field along with per-term statistics- * (such as docfreq) and pointers to the frequencies, positions, payload and skip data in the- * .doc, .pos, and .pay files. See {@code BlockTreeTermsWriter} for more details on the- * format.- * <p>NOTE: The term dictionary can plug into different postings implementations: the postings- * writer/reader are actually responsible for encoding and decoding the PostingsHeader and- * TermMetadata sections described here:- * <ul>- * <li>PostingsHeader --> Header, PackedBlockSize- * <li>TermMetadata --> (DocFPDelta|SingletonDocID), PosFPDelta?, PosVIntBlockFPDelta?,- * PayFPDelta?, SkipFPDelta?- * <li>Header, --> {@link CodecUtil#writeIndexHeader IndexHeader}- * <li>PackedBlockSize, SingletonDocID --> {@link DataOutput#writeVInt VInt}- * <li>DocFPDelta, PosFPDelta, PayFPDelta, PosVIntBlockFPDelta, SkipFPDelta --> {@link- * DataOutput#writeVLong VLong}- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}- * </ul>- * <p>Notes:- * <ul>- * <li>Header is a {@link CodecUtil#writeIndexHeader IndexHeader} storing the version- * information for the postings.- * <li>PackedBlockSize is the fixed block size for packed blocks. In packed block, bit width- * is determined by the largest integer. Smaller block size result in smaller variance- * among width of integers hence smaller indexes. Larger block size result in more- * efficient bulk i/o hence better acceleration. This value should always be a multiple- * of 64, currently fixed as 128 as a tradeoff. It is also the skip interval used to- * accelerate {@link org.apache.lucene.index.PostingsEnum#advance(int)}.- * <li>DocFPDelta determines the position of this term's TermFreqs within the .doc file. In- * particular, it is the difference of file offset between this term's data and previous- * term's data (or zero, for the first term in the block).On disk it is stored as the- * difference from previous value in sequence.- * <li>PosFPDelta determines the position of this term's TermPositions within the .pos file.- * While PayFPDelta determines the position of this term's <TermPayloads,- * TermOffsets?> within the .pay file. Similar to DocFPDelta, it is the difference- * between two file positions (or neglected, for fields that omit payloads and offsets).- * <li>PosVIntBlockFPDelta determines the position of this term's last TermPosition in last- * pos packed block within the .pos file. It is synonym for PayVIntBlockFPDelta or- * OffsetVIntBlockFPDelta. This is actually used to indicate whether it is necessary to- * load following payloads and offsets from .pos instead of .pay. Every time a new block- * of positions are to be loaded, the PostingsReader will use this value to check- * whether current block is packed format or VInt. When packed format, payloads and- * offsets are fetched from .pay, otherwise from .pos. (this value is neglected when- * total number of positions i.e. totalTermFreq is less or equal to PackedBlockSize).- * <li>SkipFPDelta determines the position of this term's SkipData within the .doc file. In- * particular, it is the length of the TermFreq data. SkipDelta is only stored if- * DocFreq is not smaller than SkipMinimum (i.e. 128 in Lucene84PostingsFormat).- * <li>SingletonDocID is an optimization when a term only appears in one document. In this- * case, instead of writing a file pointer to the .doc file (DocFPDelta), and then a- * VIntBlock at that location, the single document ID is written to the term dictionary.- * </ul>- * </dl>- *- * <a id="Termindex"></a>- *- * <dl>- * <dd><b>Term Index</b>- * <p>The .tip file contains an index into the term dictionary, so that it can be accessed- * randomly. See {@code BlockTreeTermsWriter} for more details on the format.- * </dl>- *- * <a id="Frequencies"></a>- *- * <dl>- * <dd><b>Frequencies and Skip Data</b>- * <p>The .doc file contains the lists of documents which contain each term, along with the- * frequency of the term in that document (except when frequencies are omitted: {@link- * IndexOptions#DOCS}). It also saves skip data to the beginning of each packed or VInt block,- * when the length of document list is larger than packed block size.- * <ul>- * <li>docFile(.doc) --> Header, <TermFreqs, SkipData?><sup>TermCount</sup>, Footer- * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}- * <li>TermFreqs --> <PackedBlock> <sup>PackedDocBlockNum</sup>, VIntBlock?- * <li>PackedBlock --> PackedDocDeltaBlock, PackedFreqBlock?- * <li>VIntBlock --> <DocDelta[,- * Freq?]><sup>DocFreq-PackedBlockSize*PackedDocBlockNum</sup>- * <li>SkipData --> <<SkipLevelLength, SkipLevel> <sup>NumSkipLevels-1</sup>,- * SkipLevel>, SkipDatum?- * <li>SkipLevel --> <SkipDatum> <sup>TrimmedDocFreq/(PackedBlockSize^(Level +- * 1))</sup>- * <li>SkipDatum --> DocSkip, DocFPSkip, <PosFPSkip, PosBlockOffset, PayLength?,- * PayFPSkip?>?, SkipChildLevelPointer?- * <li>PackedDocDeltaBlock, PackedFreqBlock --> {@link PackedInts PackedInts}- * <li>DocDelta, Freq, DocSkip, DocFPSkip, PosFPSkip, PosBlockOffset, PayByteUpto, PayFPSkip- * --> {@link DataOutput#writeVInt VInt}- * <li>SkipChildLevelPointer --> {@link DataOutput#writeVLong VLong}- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}- * </ul>- * <p>Notes:- * <ul>- * <li>PackedDocDeltaBlock is theoretically generated from two steps:- * <ol>- * <li>Calculate the difference between each document number and previous one, and get- * a d-gaps list (for the first document, use absolute value);- * <li>For those d-gaps from first one to- * PackedDocBlockNum*PackedBlockSize<sup>th</sup>, separately encode as packed- * blocks.- * </ol>- * If frequencies are not omitted, PackedFreqBlock will be generated without d-gap step.- * <li>VIntBlock stores remaining d-gaps (along with frequencies when possible) with a- * format that encodes DocDelta and Freq:- * <p>DocDelta: if frequencies are indexed, this determines both the document number and- * the frequency. In particular, DocDelta/2 is the difference between this document- * number and the previous document number (or zero when this is the first document in a- * TermFreqs). When DocDelta is odd, the frequency is one. When DocDelta is even, the- * frequency is read as another VInt. If frequencies are omitted, DocDelta contains the- * gap (not multiplied by 2) between document numbers and no frequency information is- * stored.- * <p>For example, the TermFreqs for a term which occurs once in document seven and- * three times in document eleven, with frequencies indexed, would be the following- * sequence of VInts:- * <p>15, 8, 3- * <p>If frequencies were omitted ({@link IndexOptions#DOCS}) it would be this sequence- * of VInts instead:- * <p>7,4- * <li>PackedDocBlockNum is the number of packed blocks for current term's docids or- * frequencies. In particular, PackedDocBlockNum = floor(DocFreq/PackedBlockSize)- * <li>TrimmedDocFreq = DocFreq % PackedBlockSize == 0 ? DocFreq - 1 : DocFreq. We use this- * trick since the definition of skip entry is a little different from base interface.- * In {@link MultiLevelSkipListWriter}, skip data is assumed to be saved for- * skipInterval<sup>th</sup>, 2*skipInterval<sup>th</sup> ... posting in the list.- * However, in Lucene84PostingsFormat, the skip data is saved for- * skipInterval+1<sup>th</sup>, 2*skipInterval+1<sup>th</sup> ... posting- * (skipInterval==PackedBlockSize in this case). When DocFreq is multiple of- * PackedBlockSize, MultiLevelSkipListWriter will expect one more skip data than- * Lucene84SkipWriter.- * <li>SkipDatum is the metadata of one skip entry. For the first block (no matter packed or- * VInt), it is omitted.- * <li>DocSkip records the document number of every PackedBlockSize<sup>th</sup> document- * number in the postings (i.e. last document number in each packed block). On disk it- * is stored as the difference from previous value in the sequence.- * <li>DocFPSkip records the file offsets of each block (excluding )posting at- * PackedBlockSize+1<sup>th</sup>, 2*PackedBlockSize+1<sup>th</sup> ... , in DocFile.- * The file offsets are relative to the start of current term's TermFreqs. On disk it is- * also stored as the difference from previous SkipDatum in the sequence.- * <li>Since positions and payloads are also block encoded, the skip should skip to related- * block first, then fetch the values according to in-block offset. PosFPSkip and- * PayFPSkip record the file offsets of related block in .pos and .pay, respectively.- * While PosBlockOffset indicates which value to fetch inside the related block- * (PayBlockOffset is unnecessary since it is always equal to PosBlockOffset). Same as- * DocFPSkip, the file offsets are relative to the start of current term's TermFreqs,- * and stored as a difference sequence.- * <li>PayByteUpto indicates the start offset of the current payload. It is equivalent to- * the sum of the payload lengths in the current block up to PosBlockOffset- * </ul>- * </dl>- *- * <a id="Positions"></a>- *- * <dl>- * <dd><b>Positions</b>- * <p>The .pos file contains the lists of positions that each term occurs at within documents.- * It also sometimes stores part of payloads and offsets for speedup.- * <ul>- * <li>PosFile(.pos) --> Header, <TermPositions> <sup>TermCount</sup>, Footer- * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}- * <li>TermPositions --> <PackedPosDeltaBlock> <sup>PackedPosBlockNum</sup>,- * VIntBlock?- * <li>VIntBlock --> <PositionDelta[, PayloadLength?], PayloadData?, OffsetDelta?,- * OffsetLength?><sup>PosVIntCount</sup>- * <li>PackedPosDeltaBlock --> {@link PackedInts PackedInts}- * <li>PositionDelta, OffsetDelta, OffsetLength --> {@link DataOutput#writeVInt VInt}- * <li>PayloadData --> {@link DataOutput#writeByte byte}<sup>PayLength</sup>- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}- * </ul>- * <p>Notes:- * <ul>- * <li>TermPositions are order by term (terms are implicit, from the term dictionary), and- * position values for each term document pair are incremental, and ordered by document- * number.- * <li>PackedPosBlockNum is the number of packed blocks for current term's positions,- * payloads or offsets. In particular, PackedPosBlockNum =- * floor(totalTermFreq/PackedBlockSize)- * <li>PosVIntCount is the number of positions encoded as VInt format. In particular,- * PosVIntCount = totalTermFreq - PackedPosBlockNum*PackedBlockSize- * <li>The procedure how PackedPosDeltaBlock is generated is the same as PackedDocDeltaBlock- * in chapter <a href="#Frequencies">Frequencies and Skip Data</a>.- * <li>PositionDelta is, if payloads are disabled for the term's field, the difference- * between the position of the current occurrence in the document and the previous- * occurrence (or zero, if this is the first occurrence in this document). If payloads- * are enabled for the term's field, then PositionDelta/2 is the difference between the- * current and the previous position. If payloads are enabled and PositionDelta is odd,- * then PayloadLength is stored, indicating the length of the payload at the current- * term position.- * <li>For example, the TermPositions for a term which occurs as the fourth term in one- * document, and as the fifth and ninth term in a subsequent document, would be the- * following sequence of VInts (payloads disabled):- * <p>4, 5, 4- * <li>PayloadData is metadata associated with the current term position. If PayloadLength- * is stored at the current position, then it indicates the length of this payload. If- * PayloadLength is not stored, then this payload has the same length as the payload at- * the previous position.- * <li>OffsetDelta/2 is the difference between this position's startOffset from the previous- * occurrence (or zero, if this is the first occurrence in this document). If- * OffsetDelta is odd, then the length (endOffset-startOffset) differs from the previous- * occurrence and an OffsetLength follows. Offset data is only written for {@link- * IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}.- * </ul>- * </dl>- *- * <a id="Payloads"></a>- *- * <dl>- * <dd><b>Payloads and Offsets</b>- * <p>The .pay file will store payloads and offsets associated with certain term-document- * positions. Some payloads and offsets will be separated out into .pos file, for performance- * reasons.- * <ul>- * <li>PayFile(.pay): --> Header, <TermPayloads, TermOffsets?>- * <sup>TermCount</sup>, Footer- * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}- * <li>TermPayloads --> <PackedPayLengthBlock, SumPayLength, PayData>- * <sup>PackedPayBlockNum</sup>- * <li>TermOffsets --> <PackedOffsetStartDeltaBlock, PackedOffsetLengthBlock>- * <sup>PackedPayBlockNum</sup>- * <li>PackedPayLengthBlock, PackedOffsetStartDeltaBlock, PackedOffsetLengthBlock -->- * {@link PackedInts PackedInts}- * <li>SumPayLength --> {@link DataOutput#writeVInt VInt}- * <li>PayData --> {@link DataOutput#writeByte byte}<sup>SumPayLength</sup>- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}- * </ul>- * <p>Notes:- * <ul>- * <li>The order of TermPayloads/TermOffsets will be the same as TermPositions, note that- * part of payload/offsets are stored in .pos.- * <li>The procedure how PackedPayLengthBlock and PackedOffsetLengthBlock are generated is- * the same as PackedFreqBlock in chapter <a href="#Frequencies">Frequencies and Skip- * Data</a>. While PackedStartDeltaBlock follows a same procedure as- * PackedDocDeltaBlock.- * <li>PackedPayBlockNum is always equal to PackedPosBlockNum, for the same term. It is also- * synonym for PackedOffsetBlockNum.- * <li>SumPayLength is the total length of payloads written within one block, should be the- * sum of PayLengths in one packed block.- * <li>PayLength in PackedPayLengthBlock is the length of each payload associated with the- * current position.- * </ul>- * </dl>- *- * @lucene.experimental- */-public class Lucene84PostingsFormat extends PostingsFormat {-- /**- * Filename extension for document number, frequencies, and skip data. See chapter: <a- * href="#Frequencies">Frequencies and Skip Data</a>- */- public static final String DOC_EXTENSION = "doc";-- /** Filename extension for positions. See chapter: <a href="#Positions">Positions</a> */- public static final String POS_EXTENSION = "pos";-- /**- * Filename extension for payloads and offsets. See chapter: <a href="#Payloads">Payloads and- * Offsets</a>- */- public static final String PAY_EXTENSION = "pay";-- /** Size of blocks. */- public static final int BLOCK_SIZE = ForUtil.BLOCK_SIZE;-- /**- * Expert: The maximum number of skip levels. Smaller values result in slightly smaller indexes,- * but slower skipping in big posting lists.- */- static final int MAX_SKIP_LEVELS = 10;-- static final String TERMS_CODEC = "Lucene84PostingsWriterTerms";- static final String DOC_CODEC = "Lucene84PostingsWriterDoc";- static final String POS_CODEC = "Lucene84PostingsWriterPos";- static final String PAY_CODEC = "Lucene84PostingsWriterPay";-- // Increment version to change it- static final int VERSION_START = 0;- // Better compression of the terms dictionary in case most terms have a docFreq of 1- static final int VERSION_COMPRESSED_TERMS_DICT_IDS = 1;- static final int VERSION_CURRENT = VERSION_COMPRESSED_TERMS_DICT_IDS;-- /** Creates {@code Lucene84PostingsFormat} with default settings. */- public Lucene84PostingsFormat() {- super("Lucene84");- }-- @Override- public String toString() {- return getName();- }-- @Override- public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {- throw new UnsupportedOperationException("Old codecs may only be used for reading");- }-- @Override- public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {- PostingsReaderBase postingsReader = new Lucene84PostingsReader(state);- boolean success = false;- try {- FieldsProducer ret = new Lucene40BlockTreeTermsReader(postingsReader, state);- success = true;- return ret;- } finally {- if (!success) {- IOUtils.closeWhileHandlingException(postingsReader);- }- }- }-- /**- * Holds all state required for {@link Lucene84PostingsReader} to produce a {@link- * org.apache.lucene.index.PostingsEnum} without re-seeking the terms dict.- *- * @lucene.internal- */- public static final class IntBlockTermState extends BlockTermState {- /** file pointer to the start of the doc ids enumeration, in {@link #DOC_EXTENSION} file */- public long docStartFP;-- /** file pointer to the start of the positions enumeration, in {@link #POS_EXTENSION} file */- public long posStartFP;-- /** file pointer to the start of the payloads enumeration, in {@link #PAY_EXTENSION} file */- public long payStartFP;-- /**- * file offset for the start of the skip list, relative to docStartFP, if there are more than- * {@link ForUtil#BLOCK_SIZE} docs; otherwise -1- */- public long skipOffset;-- /**- * file offset for the last position in the last block, if there are more than {@link- * ForUtil#BLOCK_SIZE} positions; otherwise -1- */- public long lastPosBlockOffset;-- /**- * docid when there is a single pulsed posting, otherwise -1. freq is always implicitly- * totalTermFreq in this case.- */- public int singletonDocID;-- /** Sole constructor. */- public IntBlockTermState() {- skipOffset = -1;- lastPosBlockOffset = -1;- singletonDocID = -1;- }-- @Override- public IntBlockTermState clone() {- IntBlockTermState other = new IntBlockTermState();- other.copyFrom(this);- return other;- }-- @Override- public void copyFrom(TermState _other) {- super.copyFrom(_other);- IntBlockTermState other = (IntBlockTermState) _other;- docStartFP = other.docStartFP;- posStartFP = other.posStartFP;- payStartFP = other.payStartFP;- lastPosBlockOffset = other.lastPosBlockOffset;- skipOffset = other.skipOffset;- singletonDocID = other.singletonDocID;- }-- @Override- public String toString() {- return super.toString()- + " docStartFP="- + docStartFP- + " posStartFP="- + posStartFP- + " payStartFP="- + payStartFP- + " lastPosBlockOffset="- + lastPosBlockOffset- + " singletonDocID="- + singletonDocID;- }- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84PostingsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84PostingsReader.javadeleted file mode 100644index 9709c7aa9d7..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84PostingsReader.java+++ /dev/null@@ -1,2092 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene84;--import static org.apache.lucene.backward_codecs.lucene84.ForUtil.BLOCK_SIZE;-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.DOC_CODEC;-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.MAX_SKIP_LEVELS;-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.PAY_CODEC;-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.POS_CODEC;-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.TERMS_CODEC;-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.VERSION_COMPRESSED_TERMS_DICT_IDS;-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.VERSION_CURRENT;-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.VERSION_START;--import java.io.IOException;-import java.util.Arrays;-import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.IntBlockTermState;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.BlockTermState;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.PostingsReaderBase;-import org.apache.lucene.index.FieldInfo;-import org.apache.lucene.index.Impacts;-import org.apache.lucene.index.ImpactsEnum;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.IndexOptions;-import org.apache.lucene.index.PostingsEnum;-import org.apache.lucene.index.SegmentReadState;-import org.apache.lucene.index.SlowImpactsEnum;-import org.apache.lucene.store.DataInput;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.BitUtil;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.IOUtils;--/**- * Concrete class that reads docId(maybe frq,pos,offset,payloads) list with postings format.- *- * @lucene.experimental- */-public final class Lucene84PostingsReader extends PostingsReaderBase {-- private final IndexInput docIn;- private final IndexInput posIn;- private final IndexInput payIn;-- private final int version;-- /** Sole constructor. */- public Lucene84PostingsReader(SegmentReadState state) throws IOException {- boolean success = false;- IndexInput docIn = null;- IndexInput posIn = null;- IndexInput payIn = null;-- // NOTE: these data files are too costly to verify checksum against all the bytes on open,- // but for now we at least verify proper structure of the checksum footer: which looks- // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption- // such as file truncation.-- String docName =- IndexFileNames.segmentFileName(- state.segmentInfo.name, state.segmentSuffix, Lucene84PostingsFormat.DOC_EXTENSION);- try {- docIn = EndiannessReverserUtil.openInput(state.directory, docName, state.context);- version =- CodecUtil.checkIndexHeader(- docIn,- DOC_CODEC,- VERSION_START,- VERSION_CURRENT,- state.segmentInfo.getId(),- state.segmentSuffix);- CodecUtil.retrieveChecksum(docIn);-- if (state.fieldInfos.hasProx()) {- String proxName =- IndexFileNames.segmentFileName(- state.segmentInfo.name, state.segmentSuffix, Lucene84PostingsFormat.POS_EXTENSION);- posIn = EndiannessReverserUtil.openInput(state.directory, proxName, state.context);- CodecUtil.checkIndexHeader(- posIn, POS_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);- CodecUtil.retrieveChecksum(posIn);-- if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {- String payName =- IndexFileNames.segmentFileName(- state.segmentInfo.name,- state.segmentSuffix,- Lucene84PostingsFormat.PAY_EXTENSION);- payIn = EndiannessReverserUtil.openInput(state.directory, payName, state.context);- CodecUtil.checkIndexHeader(- payIn, PAY_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);- CodecUtil.retrieveChecksum(payIn);- }- }-- this.docIn = docIn;- this.posIn = posIn;- this.payIn = payIn;- success = true;- } finally {- if (!success) {- IOUtils.closeWhileHandlingException(docIn, posIn, payIn);- }- }- }-- @Override- public void init(IndexInput termsIn, SegmentReadState state) throws IOException {- // Make sure we are talking to the matching postings writer- CodecUtil.checkIndexHeader(- termsIn,- TERMS_CODEC,- VERSION_START,- VERSION_CURRENT,- state.segmentInfo.getId(),- state.segmentSuffix);- final int indexBlockSize = termsIn.readVInt();- if (indexBlockSize != ForUtil.BLOCK_SIZE) {- throw new IllegalStateException(- "index-time BLOCK_SIZE ("- + indexBlockSize- + ") != read-time BLOCK_SIZE ("- + ForUtil.BLOCK_SIZE- + ")");- }- }-- /** Read values that have been written using variable-length encoding instead of bit-packing. */- static void readVIntBlock(- IndexInput docIn, long[] docBuffer, long[] freqBuffer, int num, boolean indexHasFreq)- throws IOException {- if (indexHasFreq) {- for (int i = 0; i < num; i++) {- final int code = docIn.readVInt();- docBuffer[i] = code >>> 1;- if ((code & 1) != 0) {- freqBuffer[i] = 1;- } else {- freqBuffer[i] = docIn.readVInt();- }- }- } else {- for (int i = 0; i < num; i++) {- docBuffer[i] = docIn.readVInt();- }- }- }-- static void prefixSum(long[] buffer, int count, long base) {- buffer[0] += base;- for (int i = 1; i < count; ++i) {- buffer[i] += buffer[i - 1];- }- }-- static int findFirstGreater(long[] buffer, int target, int from) {- for (int i = from; i < BLOCK_SIZE; ++i) {- if (buffer[i] >= target) {- return i;- }- }- return BLOCK_SIZE;- }-- @Override- public BlockTermState newTermState() {- return new IntBlockTermState();- }-- @Override- public void close() throws IOException {- IOUtils.close(docIn, posIn, payIn);- }-- @Override- public void decodeTerm(- DataInput in, FieldInfo fieldInfo, BlockTermState _termState, boolean absolute)- throws IOException {- final IntBlockTermState termState = (IntBlockTermState) _termState;- final boolean fieldHasPositions =- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;- final boolean fieldHasOffsets =- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)- >= 0;- final boolean fieldHasPayloads = fieldInfo.hasPayloads();-- if (absolute) {- termState.docStartFP = 0;- termState.posStartFP = 0;- termState.payStartFP = 0;- }-- if (version >= VERSION_COMPRESSED_TERMS_DICT_IDS) {- final long l = in.readVLong();- if ((l & 0x01) == 0) {- termState.docStartFP += l >>> 1;- if (termState.docFreq == 1) {- termState.singletonDocID = in.readVInt();- } else {- termState.singletonDocID = -1;- }- } else {- assert absolute == false;- assert termState.singletonDocID != -1;- termState.singletonDocID += BitUtil.zigZagDecode(l >>> 1);- }- } else {- termState.docStartFP += in.readVLong();- }-- if (fieldHasPositions) {- termState.posStartFP += in.readVLong();- if (fieldHasOffsets || fieldHasPayloads) {- termState.payStartFP += in.readVLong();- }- }- if (version < VERSION_COMPRESSED_TERMS_DICT_IDS) {- if (termState.docFreq == 1) {- termState.singletonDocID = in.readVInt();- } else {- termState.singletonDocID = -1;- }- }- if (fieldHasPositions) {- if (termState.totalTermFreq > BLOCK_SIZE) {- termState.lastPosBlockOffset = in.readVLong();- } else {- termState.lastPosBlockOffset = -1;- }- }- if (termState.docFreq > BLOCK_SIZE) {- termState.skipOffset = in.readVLong();- } else {- termState.skipOffset = -1;- }- }-- @Override- public PostingsEnum postings(- FieldInfo fieldInfo, BlockTermState termState, PostingsEnum reuse, int flags)- throws IOException {-- boolean indexHasPositions =- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;-- if (indexHasPositions == false- || PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS) == false) {- BlockDocsEnum docsEnum;- if (reuse instanceof BlockDocsEnum) {- docsEnum = (BlockDocsEnum) reuse;- if (!docsEnum.canReuse(docIn, fieldInfo)) {- docsEnum = new BlockDocsEnum(fieldInfo);- }- } else {- docsEnum = new BlockDocsEnum(fieldInfo);- }- return docsEnum.reset((IntBlockTermState) termState, flags);- } else {- EverythingEnum everythingEnum;- if (reuse instanceof EverythingEnum) {- everythingEnum = (EverythingEnum) reuse;- if (!everythingEnum.canReuse(docIn, fieldInfo)) {- everythingEnum = new EverythingEnum(fieldInfo);- }- } else {- everythingEnum = new EverythingEnum(fieldInfo);- }- return everythingEnum.reset((IntBlockTermState) termState, flags);- }- }-- @Override- public ImpactsEnum impacts(FieldInfo fieldInfo, BlockTermState state, int flags)- throws IOException {- if (state.docFreq <= BLOCK_SIZE) {- // no skip data- return new SlowImpactsEnum(postings(fieldInfo, state, null, flags));- }-- final boolean indexHasPositions =- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;- final boolean indexHasOffsets =- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)- >= 0;- final boolean indexHasPayloads = fieldInfo.hasPayloads();-- if (indexHasPositions == false- || PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS) == false) {- return new BlockImpactsDocsEnum(fieldInfo, (IntBlockTermState) state);- }-- if (indexHasPositions- && PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)- && (indexHasOffsets == false- || PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS) == false)- && (indexHasPayloads == false- || PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS) == false)) {- return new BlockImpactsPostingsEnum(fieldInfo, (IntBlockTermState) state);- }-- return new BlockImpactsEverythingEnum(fieldInfo, (IntBlockTermState) state, flags);- }-- final class BlockDocsEnum extends PostingsEnum {-- final ForUtil forUtil = new ForUtil();- final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);- final PForUtil pforUtil = new PForUtil(forUtil);-- private final long[] docBuffer = new long[BLOCK_SIZE + 1];- private final long[] freqBuffer = new long[BLOCK_SIZE];-- private int docBufferUpto;-- private Lucene84SkipReader skipper;- private boolean skipped;-- final IndexInput startDocIn;-- IndexInput docIn;- final boolean indexHasFreq;- final boolean indexHasPos;- final boolean indexHasOffsets;- final boolean indexHasPayloads;-- private int docFreq; // number of docs in this posting list- private long totalTermFreq; // sum of freqBuffer in this posting list (or docFreq when omitted)- private int blockUpto; // number of docs in or before the current block- private int doc; // doc we last read- private long accum; // accumulator for doc deltas-- // Where this term's postings start in the .doc file:- private long docTermStartFP;-- // Where this term's skip data starts (after- // docTermStartFP) in the .doc file (or -1 if there is- // no skip data for this term):- private long skipOffset;-- // docID for next skip point, we won't use skipper if- // target docID is not larger than this- private int nextSkipDoc;-- private boolean needsFreq; // true if the caller actually needs frequencies- // as we read freqBuffer lazily, isFreqsRead shows if freqBuffer are read for the current block- // always true when we don't have freqBuffer (indexHasFreq=false) or don't need freqBuffer- // (needsFreq=false)- private boolean isFreqsRead;- private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1-- public BlockDocsEnum(FieldInfo fieldInfo) throws IOException {- this.startDocIn = Lucene84PostingsReader.this.docIn;- this.docIn = null;- indexHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;- indexHasPos =- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;- indexHasOffsets =- fieldInfo- .getIndexOptions()- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)- >= 0;- indexHasPayloads = fieldInfo.hasPayloads();- // We set the last element of docBuffer to NO_MORE_DOCS, it helps save conditionals in- // advance()- docBuffer[BLOCK_SIZE] = NO_MORE_DOCS;- }-- public boolean canReuse(IndexInput docIn, FieldInfo fieldInfo) {- return docIn == startDocIn- && indexHasFreq- == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0)- && indexHasPos- == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)- >= 0)- && indexHasPayloads == fieldInfo.hasPayloads();- }-- public PostingsEnum reset(IntBlockTermState termState, int flags) throws IOException {- docFreq = termState.docFreq;- totalTermFreq = indexHasFreq ? termState.totalTermFreq : docFreq;- docTermStartFP = termState.docStartFP;- skipOffset = termState.skipOffset;- singletonDocID = termState.singletonDocID;- if (docFreq > 1) {- if (docIn == null) {- // lazy init- docIn = startDocIn.clone();- }- docIn.seek(docTermStartFP);- }-- doc = -1;- this.needsFreq = PostingsEnum.featureRequested(flags, PostingsEnum.FREQS);- this.isFreqsRead = true;- if (indexHasFreq == false || needsFreq == false) {- for (int i = 0; i < BLOCK_SIZE; ++i) {- freqBuffer[i] = 1;- }- }- accum = 0;- blockUpto = 0;- nextSkipDoc = BLOCK_SIZE - 1; // we won't skip if target is found in first block- docBufferUpto = BLOCK_SIZE;- skipped = false;- return this;- }-- @Override- public int freq() throws IOException {- if (isFreqsRead == false) {- pforUtil.decode(docIn, freqBuffer); // read freqBuffer for this block- isFreqsRead = true;- }- return (int) freqBuffer[docBufferUpto - 1];- }-- @Override- public int nextPosition() throws IOException {- return -1;- }-- @Override- public int startOffset() throws IOException {- return -1;- }-- @Override- public int endOffset() throws IOException {- return -1;- }-- @Override- public BytesRef getPayload() throws IOException {- return null;- }-- @Override- public int docID() {- return doc;- }-- private void refillDocs() throws IOException {- // Check if we skipped reading the previous block of freqBuffer, and if yes, position docIn- // after it- if (isFreqsRead == false) {- pforUtil.skip(docIn);- isFreqsRead = true;- }-- final int left = docFreq - blockUpto;- assert left >= 0;-- if (left >= BLOCK_SIZE) {- forDeltaUtil.decodeAndPrefixSum(docIn, accum, docBuffer);-- if (indexHasFreq) {- if (needsFreq) {- isFreqsRead = false;- } else {- pforUtil.skip(docIn); // skip over freqBuffer if we don't need them at all- }- }- blockUpto += BLOCK_SIZE;- } else if (docFreq == 1) {- docBuffer[0] = singletonDocID;- freqBuffer[0] = totalTermFreq;- docBuffer[1] = NO_MORE_DOCS;- blockUpto++;- } else {- // Read vInts:- readVIntBlock(docIn, docBuffer, freqBuffer, left, indexHasFreq);- prefixSum(docBuffer, left, accum);- docBuffer[left] = NO_MORE_DOCS;- blockUpto += left;- }- accum = docBuffer[BLOCK_SIZE - 1];- docBufferUpto = 0;- assert docBuffer[BLOCK_SIZE] == NO_MORE_DOCS;- }-- @Override- public int nextDoc() throws IOException {- if (docBufferUpto == BLOCK_SIZE) {- refillDocs(); // we don't need to load freqBuffer for now (will be loaded later if- // necessary)- }-- doc = (int) docBuffer[docBufferUpto];- docBufferUpto++;- return doc;- }-- @Override- public int advance(int target) throws IOException {- // current skip docID < docIDs generated from current buffer <= next skip docID- // we don't need to skip if target is buffered already- if (docFreq > BLOCK_SIZE && target > nextSkipDoc) {-- if (skipper == null) {- // Lazy init: first time this enum has ever been used for skipping- skipper =- new Lucene84SkipReader(- docIn.clone(), MAX_SKIP_LEVELS, indexHasPos, indexHasOffsets, indexHasPayloads);- }-- if (!skipped) {- assert skipOffset != -1;- // This is the first time this enum has skipped- // since reset() was called; load the skip data:- skipper.init(docTermStartFP + skipOffset, docTermStartFP, 0, 0, docFreq);- skipped = true;- }-- // always plus one to fix the result, since skip position in Lucene84SkipReader- // is a little different from MultiLevelSkipListReader- final int newDocUpto = skipper.skipTo(target) + 1;-- if (newDocUpto >= blockUpto) {- // Skipper moved- assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;- blockUpto = newDocUpto;-- // Force to read next block- docBufferUpto = BLOCK_SIZE;- accum = skipper.getDoc(); // actually, this is just lastSkipEntry- docIn.seek(skipper.getDocPointer()); // now point to the block we want to search- // even if freqBuffer were not read from the previous block, we will mark them as read,- // as we don't need to skip the previous block freqBuffer in refillDocs,- // as we have already positioned docIn where in needs to be.- isFreqsRead = true;- }- // next time we call advance, this is used to- // foresee whether skipper is necessary.- nextSkipDoc = skipper.getNextSkipDoc();- }- if (docBufferUpto == BLOCK_SIZE) {- refillDocs();- }-- // Now scan... this is an inlined/pared down version- // of nextDoc():- long doc;- while (true) {- doc = docBuffer[docBufferUpto];-- if (doc >= target) {- break;- }- ++docBufferUpto;- }-- docBufferUpto++;- return this.doc = (int) doc;- }-- @Override- public long cost() {- return docFreq;- }- }-- // Also handles payloads + offsets- final class EverythingEnum extends PostingsEnum {-- final ForUtil forUtil = new ForUtil();- final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);- final PForUtil pforUtil = new PForUtil(forUtil);-- private final long[] docBuffer = new long[BLOCK_SIZE + 1];- private final long[] freqBuffer = new long[BLOCK_SIZE + 1];- private final long[] posDeltaBuffer = new long[BLOCK_SIZE];-- private final long[] payloadLengthBuffer;- private final long[] offsetStartDeltaBuffer;- private final long[] offsetLengthBuffer;-- private byte[] payloadBytes;- private int payloadByteUpto;- private int payloadLength;-- private int lastStartOffset;- private int startOffset;- private int endOffset;-- private int docBufferUpto;- private int posBufferUpto;-- private Lucene84SkipReader skipper;- private boolean skipped;-- final IndexInput startDocIn;-- IndexInput docIn;- final IndexInput posIn;- final IndexInput payIn;- final BytesRef payload;-- final boolean indexHasOffsets;- final boolean indexHasPayloads;-- private int docFreq; // number of docs in this posting list- private long totalTermFreq; // number of positions in this posting list- private int blockUpto; // number of docs in or before the current block- private int doc; // doc we last read- private long accum; // accumulator for doc deltas- private int freq; // freq we last read- private int position; // current position-- // how many positions "behind" we are; nextPosition must- // skip these to "catch up":- private int posPendingCount;-- // Lazy pos seek: if != -1 then we must seek to this FP- // before reading positions:- private long posPendingFP;-- // Lazy pay seek: if != -1 then we must seek to this FP- // before reading payloads/offsets:- private long payPendingFP;-- // Where this term's postings start in the .doc file:- private long docTermStartFP;-- // Where this term's postings start in the .pos file:- private long posTermStartFP;-- // Where this term's payloads/offsets start in the .pay- // file:- private long payTermStartFP;-- // File pointer where the last (vInt encoded) pos delta- // block is. We need this to know whether to bulk- // decode vs vInt decode the block:- private long lastPosBlockFP;-- // Where this term's skip data starts (after- // docTermStartFP) in the .doc file (or -1 if there is- // no skip data for this term):- private long skipOffset;-- private int nextSkipDoc;-- private boolean needsOffsets; // true if we actually need offsets- private boolean needsPayloads; // true if we actually need payloads- private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1-- public EverythingEnum(FieldInfo fieldInfo) throws IOException {- indexHasOffsets =- fieldInfo- .getIndexOptions()- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)- >= 0;- indexHasPayloads = fieldInfo.hasPayloads();-- this.startDocIn = Lucene84PostingsReader.this.docIn;- this.docIn = null;- this.posIn = Lucene84PostingsReader.this.posIn.clone();- if (indexHasOffsets || indexHasPayloads) {- this.payIn = Lucene84PostingsReader.this.payIn.clone();- } else {- this.payIn = null;- }- if (indexHasOffsets) {- offsetStartDeltaBuffer = new long[BLOCK_SIZE];- offsetLengthBuffer = new long[BLOCK_SIZE];- } else {- offsetStartDeltaBuffer = null;- offsetLengthBuffer = null;- startOffset = -1;- endOffset = -1;- }-- if (indexHasPayloads) {- payloadLengthBuffer = new long[BLOCK_SIZE];- payloadBytes = new byte[128];- payload = new BytesRef();- } else {- payloadLengthBuffer = null;- payloadBytes = null;- payload = null;- }-- // We set the last element of docBuffer to NO_MORE_DOCS, it helps save conditionals in- // advance()- docBuffer[BLOCK_SIZE] = NO_MORE_DOCS;- }-- public boolean canReuse(IndexInput docIn, FieldInfo fieldInfo) {- return docIn == startDocIn- && indexHasOffsets- == (fieldInfo- .getIndexOptions()- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)- >= 0)- && indexHasPayloads == fieldInfo.hasPayloads();- }-- public EverythingEnum reset(IntBlockTermState termState, int flags) throws IOException {- docFreq = termState.docFreq;- docTermStartFP = termState.docStartFP;- posTermStartFP = termState.posStartFP;- payTermStartFP = termState.payStartFP;- skipOffset = termState.skipOffset;- totalTermFreq = termState.totalTermFreq;- singletonDocID = termState.singletonDocID;- if (docFreq > 1) {- if (docIn == null) {- // lazy init- docIn = startDocIn.clone();- }- docIn.seek(docTermStartFP);- }- posPendingFP = posTermStartFP;- payPendingFP = payTermStartFP;- posPendingCount = 0;- if (termState.totalTermFreq < BLOCK_SIZE) {- lastPosBlockFP = posTermStartFP;- } else if (termState.totalTermFreq == BLOCK_SIZE) {- lastPosBlockFP = -1;- } else {- lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset;- }-- this.needsOffsets = PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS);- this.needsPayloads = PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS);-- doc = -1;- accum = 0;- blockUpto = 0;- if (docFreq > BLOCK_SIZE) {- nextSkipDoc = BLOCK_SIZE - 1; // we won't skip if target is found in first block- } else {- nextSkipDoc = NO_MORE_DOCS; // not enough docs for skipping- }- docBufferUpto = BLOCK_SIZE;- skipped = false;- return this;- }-- @Override- public int freq() throws IOException {- return freq;- }-- @Override- public int docID() {- return doc;- }-- private void refillDocs() throws IOException {- final int left = docFreq - blockUpto;- assert left >= 0;-- if (left >= BLOCK_SIZE) {- forDeltaUtil.decodeAndPrefixSum(docIn, accum, docBuffer);- pforUtil.decode(docIn, freqBuffer);- blockUpto += BLOCK_SIZE;- } else if (docFreq == 1) {- docBuffer[0] = singletonDocID;- freqBuffer[0] = totalTermFreq;- docBuffer[1] = NO_MORE_DOCS;- blockUpto++;- } else {- readVIntBlock(docIn, docBuffer, freqBuffer, left, true);- prefixSum(docBuffer, left, accum);- docBuffer[left] = NO_MORE_DOCS;- blockUpto += left;- }- accum = docBuffer[BLOCK_SIZE - 1];- docBufferUpto = 0;- assert docBuffer[BLOCK_SIZE] == NO_MORE_DOCS;- }-- private void refillPositions() throws IOException {- if (posIn.getFilePointer() == lastPosBlockFP) {- final int count = (int) (totalTermFreq % BLOCK_SIZE);- int payloadLength = 0;- int offsetLength = 0;- payloadByteUpto = 0;- for (int i = 0; i < count; i++) {- int code = posIn.readVInt();- if (indexHasPayloads) {- if ((code & 1) != 0) {- payloadLength = posIn.readVInt();- }- payloadLengthBuffer[i] = payloadLength;- posDeltaBuffer[i] = code >>> 1;- if (payloadLength != 0) {- if (payloadByteUpto + payloadLength > payloadBytes.length) {- payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payloadLength);- }- posIn.readBytes(payloadBytes, payloadByteUpto, payloadLength);- payloadByteUpto += payloadLength;- }- } else {- posDeltaBuffer[i] = code;- }-- if (indexHasOffsets) {- int deltaCode = posIn.readVInt();- if ((deltaCode & 1) != 0) {- offsetLength = posIn.readVInt();- }- offsetStartDeltaBuffer[i] = deltaCode >>> 1;- offsetLengthBuffer[i] = offsetLength;- }- }- payloadByteUpto = 0;- } else {- pforUtil.decode(posIn, posDeltaBuffer);-- if (indexHasPayloads) {- if (needsPayloads) {- pforUtil.decode(payIn, payloadLengthBuffer);- int numBytes = payIn.readVInt();-- if (numBytes > payloadBytes.length) {- payloadBytes = ArrayUtil.grow(payloadBytes, numBytes);- }- payIn.readBytes(payloadBytes, 0, numBytes);- } else {- // this works, because when writing a vint block we always force the first length to be- // written- pforUtil.skip(payIn); // skip over lengths- int numBytes = payIn.readVInt(); // read length of payloadBytes- payIn.seek(payIn.getFilePointer() + numBytes); // skip over payloadBytes- }- payloadByteUpto = 0;- }-- if (indexHasOffsets) {- if (needsOffsets) {- pforUtil.decode(payIn, offsetStartDeltaBuffer);- pforUtil.decode(payIn, offsetLengthBuffer);- } else {- // this works, because when writing a vint block we always force the first length to be- // written- pforUtil.skip(payIn); // skip over starts- pforUtil.skip(payIn); // skip over lengths- }- }- }- }-- @Override- public int nextDoc() throws IOException {- if (docBufferUpto == BLOCK_SIZE) {- refillDocs();- }-- doc = (int) docBuffer[docBufferUpto];- freq = (int) freqBuffer[docBufferUpto];- posPendingCount += freq;- docBufferUpto++;-- position = 0;- lastStartOffset = 0;- return doc;- }-- @Override- public int advance(int target) throws IOException {- if (target > nextSkipDoc) {- if (skipper == null) {- // Lazy init: first time this enum has ever been used for skipping- skipper =- new Lucene84SkipReader(- docIn.clone(), MAX_SKIP_LEVELS, true, indexHasOffsets, indexHasPayloads);- }-- if (!skipped) {- assert skipOffset != -1;- // This is the first time this enum has skipped- // since reset() was called; load the skip data:- skipper.init(- docTermStartFP + skipOffset, docTermStartFP, posTermStartFP, payTermStartFP, docFreq);- skipped = true;- }-- final int newDocUpto = skipper.skipTo(target) + 1;-- if (newDocUpto > blockUpto - BLOCK_SIZE + docBufferUpto) {- // Skipper moved- assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;- blockUpto = newDocUpto;-- // Force to read next block- docBufferUpto = BLOCK_SIZE;- accum = skipper.getDoc();- docIn.seek(skipper.getDocPointer());- posPendingFP = skipper.getPosPointer();- payPendingFP = skipper.getPayPointer();- posPendingCount = skipper.getPosBufferUpto();- lastStartOffset = 0; // new document- payloadByteUpto = skipper.getPayloadByteUpto();- }- nextSkipDoc = skipper.getNextSkipDoc();- }- if (docBufferUpto == BLOCK_SIZE) {- refillDocs();- }-- // Now scan:- long doc;- while (true) {- doc = docBuffer[docBufferUpto];- freq = (int) freqBuffer[docBufferUpto];- posPendingCount += freq;- docBufferUpto++;-- if (doc >= target) {- break;- }- }-- position = 0;- lastStartOffset = 0;- return this.doc = (int) doc;- }-- // TODO: in theory we could avoid loading frq block- // when not needed, ie, use skip data to load how far to- // seek the pos pointer ... instead of having to load frq- // blocks only to sum up how many positions to skip- private void skipPositions() throws IOException {- // Skip positions now:- int toSkip = posPendingCount - freq;- // if (DEBUG) {- // System.out.println(" FPR.skipPositions: toSkip=" + toSkip);- // }-- final int leftInBlock = BLOCK_SIZE - posBufferUpto;- if (toSkip < leftInBlock) {- int end = posBufferUpto + toSkip;- while (posBufferUpto < end) {- if (indexHasPayloads) {- payloadByteUpto += payloadLengthBuffer[posBufferUpto];- }- posBufferUpto++;- }- } else {- toSkip -= leftInBlock;- while (toSkip >= BLOCK_SIZE) {- assert posIn.getFilePointer() != lastPosBlockFP;- pforUtil.skip(posIn);-- if (indexHasPayloads) {- // Skip payloadLength block:- pforUtil.skip(payIn);-- // Skip payloadBytes block:- int numBytes = payIn.readVInt();- payIn.seek(payIn.getFilePointer() + numBytes);- }-- if (indexHasOffsets) {- pforUtil.skip(payIn);- pforUtil.skip(payIn);- }- toSkip -= BLOCK_SIZE;- }- refillPositions();- payloadByteUpto = 0;- posBufferUpto = 0;- while (posBufferUpto < toSkip) {- if (indexHasPayloads) {- payloadByteUpto += payloadLengthBuffer[posBufferUpto];- }- posBufferUpto++;- }- }-- position = 0;- lastStartOffset = 0;- }-- @Override- public int nextPosition() throws IOException {- assert posPendingCount > 0;-- if (posPendingFP != -1) {- posIn.seek(posPendingFP);- posPendingFP = -1;-- if (payPendingFP != -1 && payIn != null) {- payIn.seek(payPendingFP);- payPendingFP = -1;- }-- // Force buffer refill:- posBufferUpto = BLOCK_SIZE;- }-- if (posPendingCount > freq) {- skipPositions();- posPendingCount = freq;- }-- if (posBufferUpto == BLOCK_SIZE) {- refillPositions();- posBufferUpto = 0;- }- position += posDeltaBuffer[posBufferUpto];-- if (indexHasPayloads) {- payloadLength = (int) payloadLengthBuffer[posBufferUpto];- payload.bytes = payloadBytes;- payload.offset = payloadByteUpto;- payload.length = payloadLength;- payloadByteUpto += payloadLength;- }-- if (indexHasOffsets) {- startOffset = lastStartOffset + (int) offsetStartDeltaBuffer[posBufferUpto];- endOffset = startOffset + (int) offsetLengthBuffer[posBufferUpto];- lastStartOffset = startOffset;- }-- posBufferUpto++;- posPendingCount--;- return position;- }-- @Override- public int startOffset() {- return startOffset;- }-- @Override- public int endOffset() {- return endOffset;- }-- @Override- public BytesRef getPayload() {- if (payloadLength == 0) {- return null;- } else {- return payload;- }- }-- @Override- public long cost() {- return docFreq;- }- }-- final class BlockImpactsDocsEnum extends ImpactsEnum {-- final ForUtil forUtil = new ForUtil();- final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);- final PForUtil pforUtil = new PForUtil(forUtil);-- private final long[] docBuffer = new long[BLOCK_SIZE + 1];- private final long[] freqBuffer = new long[BLOCK_SIZE];-- private int docBufferUpto;-- private final Lucene84ScoreSkipReader skipper;-- final IndexInput docIn;-- final boolean indexHasFreqs;-- private int docFreq; // number of docs in this posting list- private int blockUpto; // number of documents in or before the current block- private int doc; // doc we last read- private long accum; // accumulator for doc deltas-- private int nextSkipDoc = -1;-- private long seekTo = -1;-- // as we read freqBuffer lazily, isFreqsRead shows if freqBuffer are read for the current block- // always true when we don't have freqBuffer (indexHasFreq=false) or don't need freqBuffer- // (needsFreq=false)- private boolean isFreqsRead;-- public BlockImpactsDocsEnum(FieldInfo fieldInfo, IntBlockTermState termState)- throws IOException {- indexHasFreqs = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;- final boolean indexHasPositions =- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;- final boolean indexHasOffsets =- fieldInfo- .getIndexOptions()- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)- >= 0;- final boolean indexHasPayloads = fieldInfo.hasPayloads();-- this.docIn = Lucene84PostingsReader.this.docIn.clone();-- docFreq = termState.docFreq;- docIn.seek(termState.docStartFP);-- doc = -1;- accum = 0;- blockUpto = 0;- docBufferUpto = BLOCK_SIZE;-- skipper =- new Lucene84ScoreSkipReader(- docIn.clone(), MAX_SKIP_LEVELS, indexHasPositions, indexHasOffsets, indexHasPayloads);- skipper.init(- termState.docStartFP + termState.skipOffset,- termState.docStartFP,- termState.posStartFP,- termState.payStartFP,- docFreq);-- // We set the last element of docBuffer to NO_MORE_DOCS, it helps save conditionals in- // advance()- docBuffer[BLOCK_SIZE] = NO_MORE_DOCS;- this.isFreqsRead = true;- if (indexHasFreqs == false) {- Arrays.fill(freqBuffer, 1L);- }- }-- @Override- public int freq() throws IOException {- if (isFreqsRead == false) {- pforUtil.decode(docIn, freqBuffer); // read freqBuffer for this block- isFreqsRead = true;- }- return (int) freqBuffer[docBufferUpto - 1];- }-- @Override- public int docID() {- return doc;- }-- private void refillDocs() throws IOException {- // Check if we skipped reading the previous block of freqBuffer, and if yes, position docIn- // after it- if (isFreqsRead == false) {- pforUtil.skip(docIn);- isFreqsRead = true;- }-- final int left = docFreq - blockUpto;- assert left >= 0;-- if (left >= BLOCK_SIZE) {- forDeltaUtil.decodeAndPrefixSum(docIn, accum, docBuffer);- if (indexHasFreqs) {- pforUtil.decode(docIn, freqBuffer);- }- blockUpto += BLOCK_SIZE;- } else {- readVIntBlock(docIn, docBuffer, freqBuffer, left, indexHasFreqs);- prefixSum(docBuffer, left, accum);- docBuffer[left] = NO_MORE_DOCS;- blockUpto += left;- }- accum = docBuffer[BLOCK_SIZE - 1];- docBufferUpto = 0;- assert docBuffer[BLOCK_SIZE] == NO_MORE_DOCS;- }-- @Override- public void advanceShallow(int target) throws IOException {- if (target > nextSkipDoc) {- // always plus one to fix the result, since skip position in Lucene84SkipReader- // is a little different from MultiLevelSkipListReader- final int newDocUpto = skipper.skipTo(target) + 1;-- if (newDocUpto >= blockUpto) {- // Skipper moved- assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;- blockUpto = newDocUpto;-- // Force to read next block- docBufferUpto = BLOCK_SIZE;- accum = skipper.getDoc();- seekTo = skipper.getDocPointer(); // delay the seek- }- // next time we call advance, this is used to- // foresee whether skipper is necessary.- nextSkipDoc = skipper.getNextSkipDoc();- }- assert nextSkipDoc >= target;- }-- @Override- public Impacts getImpacts() throws IOException {- advanceShallow(doc);- return skipper.getImpacts();- }-- @Override- public int nextDoc() throws IOException {- return advance(doc + 1);- }-- @Override- public int advance(int target) throws IOException {- if (target > nextSkipDoc) {- advanceShallow(target);- }- if (docBufferUpto == BLOCK_SIZE) {- if (seekTo >= 0) {- docIn.seek(seekTo);- isFreqsRead = true; // reset isFreqsRead- seekTo = -1;- }- refillDocs();- }-- int next = findFirstGreater(docBuffer, target, docBufferUpto);- this.doc = (int) docBuffer[next];- docBufferUpto = next + 1;- return doc;- }-- @Override- public int nextPosition() throws IOException {- return -1;- }-- @Override- public int startOffset() {- return -1;- }-- @Override- public int endOffset() {- return -1;- }-- @Override- public BytesRef getPayload() {- return null;- }-- @Override- public long cost() {- return docFreq;- }- }-- final class BlockImpactsPostingsEnum extends ImpactsEnum {-- final ForUtil forUtil = new ForUtil();- final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);- final PForUtil pforUtil = new PForUtil(forUtil);-- private final long[] docBuffer = new long[BLOCK_SIZE];- private final long[] freqBuffer = new long[BLOCK_SIZE];- private final long[] posDeltaBuffer = new long[BLOCK_SIZE];-- private int docBufferUpto;- private int posBufferUpto;-- private final Lucene84ScoreSkipReader skipper;-- final IndexInput docIn;- final IndexInput posIn;-- final boolean indexHasOffsets;- final boolean indexHasPayloads;-- private int docFreq; // number of docs in this posting list- private long totalTermFreq; // number of positions in this posting list- private int docUpto; // how many docs we've read- private int doc; // doc we last read- private long accum; // accumulator for doc deltas- private int freq; // freq we last read- private int position; // current position-- // how many positions "behind" we are; nextPosition must- // skip these to "catch up":- private int posPendingCount;-- // Lazy pos seek: if != -1 then we must seek to this FP- // before reading positions:- private long posPendingFP;-- // Where this term's postings start in the .doc file:- private long docTermStartFP;-- // Where this term's postings start in the .pos file:- private long posTermStartFP;-- // Where this term's payloads/offsets start in the .pay- // file:- private long payTermStartFP;-- // File pointer where the last (vInt encoded) pos delta- // block is. We need this to know whether to bulk- // decode vs vInt decode the block:- private long lastPosBlockFP;-- private int nextSkipDoc = -1;-- private long seekTo = -1;-- public BlockImpactsPostingsEnum(FieldInfo fieldInfo, IntBlockTermState termState)- throws IOException {- indexHasOffsets =- fieldInfo- .getIndexOptions()- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)- >= 0;- indexHasPayloads = fieldInfo.hasPayloads();-- this.docIn = Lucene84PostingsReader.this.docIn.clone();-- this.posIn = Lucene84PostingsReader.this.posIn.clone();-- docFreq = termState.docFreq;- docTermStartFP = termState.docStartFP;- posTermStartFP = termState.posStartFP;- payTermStartFP = termState.payStartFP;- totalTermFreq = termState.totalTermFreq;- docIn.seek(docTermStartFP);- posPendingFP = posTermStartFP;- posPendingCount = 0;- if (termState.totalTermFreq < BLOCK_SIZE) {- lastPosBlockFP = posTermStartFP;- } else if (termState.totalTermFreq == BLOCK_SIZE) {- lastPosBlockFP = -1;- } else {- lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset;- }-- doc = -1;- accum = 0;- docUpto = 0;- docBufferUpto = BLOCK_SIZE;-- skipper =- new Lucene84ScoreSkipReader(- docIn.clone(), MAX_SKIP_LEVELS, true, indexHasOffsets, indexHasPayloads);- skipper.init(- docTermStartFP + termState.skipOffset,- docTermStartFP,- posTermStartFP,- payTermStartFP,- docFreq);- }-- @Override- public int freq() throws IOException {- return freq;- }-- @Override- public int docID() {- return doc;- }-- private void refillDocs() throws IOException {- final int left = docFreq - docUpto;- assert left >= 0;-- if (left >= BLOCK_SIZE) {- forDeltaUtil.decodeAndPrefixSum(docIn, accum, docBuffer);- pforUtil.decode(docIn, freqBuffer);- } else {- readVIntBlock(docIn, docBuffer, freqBuffer, left, true);- prefixSum(docBuffer, left, accum);- docBuffer[left] = NO_MORE_DOCS;- }- accum = docBuffer[BLOCK_SIZE - 1];- docBufferUpto = 0;- }-- private void refillPositions() throws IOException {- if (posIn.getFilePointer() == lastPosBlockFP) {- final int count = (int) (totalTermFreq % BLOCK_SIZE);- int payloadLength = 0;- for (int i = 0; i < count; i++) {- int code = posIn.readVInt();- if (indexHasPayloads) {- if ((code & 1) != 0) {- payloadLength = posIn.readVInt();- }- posDeltaBuffer[i] = code >>> 1;- if (payloadLength != 0) {- posIn.seek(posIn.getFilePointer() + payloadLength);- }- } else {- posDeltaBuffer[i] = code;- }- if (indexHasOffsets) {- if ((posIn.readVInt() & 1) != 0) {- // offset length changed- posIn.readVInt();- }- }- }- } else {- pforUtil.decode(posIn, posDeltaBuffer);- }- }-- @Override- public void advanceShallow(int target) throws IOException {- if (target > nextSkipDoc) {- // always plus one to fix the result, since skip position in Lucene84SkipReader- // is a little different from MultiLevelSkipListReader- final int newDocUpto = skipper.skipTo(target) + 1;-- if (newDocUpto > docUpto) {- // Skipper moved- assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;- docUpto = newDocUpto;-- // Force to read next block- docBufferUpto = BLOCK_SIZE;- accum = skipper.getDoc();- posPendingFP = skipper.getPosPointer();- posPendingCount = skipper.getPosBufferUpto();- seekTo = skipper.getDocPointer(); // delay the seek- }- // next time we call advance, this is used to- // foresee whether skipper is necessary.- nextSkipDoc = skipper.getNextSkipDoc();- }- assert nextSkipDoc >= target;- }-- @Override- public Impacts getImpacts() throws IOException {- advanceShallow(doc);- return skipper.getImpacts();- }-- @Override- public int nextDoc() throws IOException {- return advance(doc + 1);- }-- @Override- public int advance(int target) throws IOException {- if (target > nextSkipDoc) {- advanceShallow(target);- }- if (docBufferUpto == BLOCK_SIZE) {- if (seekTo >= 0) {- docIn.seek(seekTo);- seekTo = -1;- }- refillDocs();- }-- int next = findFirstGreater(docBuffer, target, docBufferUpto);- if (next == BLOCK_SIZE) {- return doc = NO_MORE_DOCS;- }- this.doc = (int) docBuffer[next];- this.freq = (int) freqBuffer[next];- for (int i = docBufferUpto; i <= next; ++i) {- posPendingCount += freqBuffer[i];- }- docUpto += next - docBufferUpto + 1;- docBufferUpto = next + 1;- position = 0;- return doc;- }-- // TODO: in theory we could avoid loading frq block- // when not needed, ie, use skip data to load how far to- // seek the pos pointer ... instead of having to load frq- // blocks only to sum up how many positions to skip- private void skipPositions() throws IOException {- // Skip positions now:- int toSkip = posPendingCount - freq;-- final int leftInBlock = BLOCK_SIZE - posBufferUpto;- if (toSkip < leftInBlock) {- posBufferUpto += toSkip;- } else {- toSkip -= leftInBlock;- while (toSkip >= BLOCK_SIZE) {- assert posIn.getFilePointer() != lastPosBlockFP;- pforUtil.skip(posIn);- toSkip -= BLOCK_SIZE;- }- refillPositions();- posBufferUpto = toSkip;- }-- position = 0;- }-- @Override- public int nextPosition() throws IOException {- assert posPendingCount > 0;-- if (posPendingFP != -1) {- posIn.seek(posPendingFP);- posPendingFP = -1;-- // Force buffer refill:- posBufferUpto = BLOCK_SIZE;- }-- if (posPendingCount > freq) {- skipPositions();- posPendingCount = freq;- }-- if (posBufferUpto == BLOCK_SIZE) {- refillPositions();- posBufferUpto = 0;- }- position += posDeltaBuffer[posBufferUpto++];-- posPendingCount--;- return position;- }-- @Override- public int startOffset() {- return -1;- }-- @Override- public int endOffset() {- return -1;- }-- @Override- public BytesRef getPayload() {- return null;- }-- @Override- public long cost() {- return docFreq;- }- }-- final class BlockImpactsEverythingEnum extends ImpactsEnum {-- final ForUtil forUtil = new ForUtil();- final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);- final PForUtil pforUtil = new PForUtil(forUtil);-- private final long[] docBuffer = new long[BLOCK_SIZE];- private final long[] freqBuffer = new long[BLOCK_SIZE];- private final long[] posDeltaBuffer = new long[BLOCK_SIZE];-- private final long[] payloadLengthBuffer;- private final long[] offsetStartDeltaBuffer;- private final long[] offsetLengthBuffer;-- private byte[] payloadBytes;- private int payloadByteUpto;- private int payloadLength;-- private int lastStartOffset;- private int startOffset = -1;- private int endOffset = -1;-- private int docBufferUpto;- private int posBufferUpto;-- private final Lucene84ScoreSkipReader skipper;-- final IndexInput docIn;- final IndexInput posIn;- final IndexInput payIn;- final BytesRef payload;-- final boolean indexHasFreq;- final boolean indexHasPos;- final boolean indexHasOffsets;- final boolean indexHasPayloads;-- private int docFreq; // number of docs in this posting list- private long totalTermFreq; // number of positions in this posting list- private int docUpto; // how many docs we've read- private int posDocUpTo; // for how many docs we've read positions, offsets, and payloads- private int doc; // doc we last read- private long accum; // accumulator for doc deltas- private int position; // current position-- // how many positions "behind" we are; nextPosition must- // skip these to "catch up":- private int posPendingCount;-- // Lazy pos seek: if != -1 then we must seek to this FP- // before reading positions:- private long posPendingFP;-- // Lazy pay seek: if != -1 then we must seek to this FP- // before reading payloads/offsets:- private long payPendingFP;-- // Where this term's postings start in the .doc file:- private long docTermStartFP;-- // Where this term's postings start in the .pos file:- private long posTermStartFP;-- // Where this term's payloads/offsets start in the .pay- // file:- private long payTermStartFP;-- // File pointer where the last (vInt encoded) pos delta- // block is. We need this to know whether to bulk- // decode vs vInt decode the block:- private long lastPosBlockFP;-- private int nextSkipDoc = -1;-- private final boolean needsPositions;- private final boolean needsOffsets; // true if we actually need offsets- private final boolean needsPayloads; // true if we actually need payloads-- private boolean- isFreqsRead; // shows if freqBuffer for the current doc block are read into freqBuffer-- private long seekTo = -1;-- public BlockImpactsEverythingEnum(FieldInfo fieldInfo, IntBlockTermState termState, int flags)- throws IOException {- indexHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;- indexHasPos =- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;- indexHasOffsets =- fieldInfo- .getIndexOptions()- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)- >= 0;- indexHasPayloads = fieldInfo.hasPayloads();-- needsPositions = PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS);- needsOffsets = PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS);- needsPayloads = PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS);-- this.docIn = Lucene84PostingsReader.this.docIn.clone();-- if (indexHasPos && needsPositions) {- this.posIn = Lucene84PostingsReader.this.posIn.clone();- } else {- this.posIn = null;- }-- if ((indexHasOffsets && needsOffsets) || (indexHasPayloads && needsPayloads)) {- this.payIn = Lucene84PostingsReader.this.payIn.clone();- } else {- this.payIn = null;- }-- if (indexHasOffsets) {- offsetStartDeltaBuffer = new long[BLOCK_SIZE];- offsetLengthBuffer = new long[BLOCK_SIZE];- } else {- offsetStartDeltaBuffer = null;- offsetLengthBuffer = null;- startOffset = -1;- endOffset = -1;- }-- if (indexHasPayloads) {- payloadLengthBuffer = new long[BLOCK_SIZE];- payloadBytes = new byte[128];- payload = new BytesRef();- } else {- payloadLengthBuffer = null;- payloadBytes = null;- payload = null;- }-- docFreq = termState.docFreq;- docTermStartFP = termState.docStartFP;- posTermStartFP = termState.posStartFP;- payTermStartFP = termState.payStartFP;- totalTermFreq = termState.totalTermFreq;- docIn.seek(docTermStartFP);- posPendingFP = posTermStartFP;- payPendingFP = payTermStartFP;- posPendingCount = 0;- if (termState.totalTermFreq < BLOCK_SIZE) {- lastPosBlockFP = posTermStartFP;- } else if (termState.totalTermFreq == BLOCK_SIZE) {- lastPosBlockFP = -1;- } else {- lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset;- }-- doc = -1;- accum = 0;- docUpto = 0;- posDocUpTo = 0;- isFreqsRead = true;- docBufferUpto = BLOCK_SIZE;-- skipper =- new Lucene84ScoreSkipReader(- docIn.clone(), MAX_SKIP_LEVELS, indexHasPos, indexHasOffsets, indexHasPayloads);- skipper.init(- docTermStartFP + termState.skipOffset,- docTermStartFP,- posTermStartFP,- payTermStartFP,- docFreq);-- if (indexHasFreq == false) {- for (int i = 0; i < BLOCK_SIZE; ++i) {- freqBuffer[i] = 1;- }- }- }-- @Override- public int freq() throws IOException {- if (indexHasFreq && (isFreqsRead == false)) {- pforUtil.decode(docIn, freqBuffer); // read freqBuffer for this block- isFreqsRead = true;- }- return (int) freqBuffer[docBufferUpto - 1];- }-- @Override- public int docID() {- return doc;- }-- private void refillDocs() throws IOException {- if (indexHasFreq) {- if (isFreqsRead == false) { // previous freq block was not read- // check if we need to load the previous freq block to catch up on positions or we can- // skip it- if (indexHasPos && needsPositions && (posDocUpTo < docUpto)) {- pforUtil.decode(docIn, freqBuffer); // load the previous freq block- } else {- pforUtil.skip(docIn); // skip it- }- isFreqsRead = true;- }- if (indexHasPos && needsPositions) {- while (posDocUpTo- < docUpto) { // catch on positions, bring posPendingCount upto the current doc- posPendingCount += freqBuffer[docBufferUpto - (docUpto - posDocUpTo)];- posDocUpTo++;- }- }- }-- final int left = docFreq - docUpto;- assert left >= 0;-- if (left >= BLOCK_SIZE) {- forDeltaUtil.decodeAndPrefixSum(docIn, accum, docBuffer);- if (indexHasFreq) {- isFreqsRead =- false; // freq block will be loaded lazily when necessary, we don't load it here- }- } else {- readVIntBlock(docIn, docBuffer, freqBuffer, left, indexHasFreq);- prefixSum(docBuffer, left, accum);- docBuffer[left] = NO_MORE_DOCS;- }- accum = docBuffer[BLOCK_SIZE - 1];- docBufferUpto = 0;- }-- private void refillPositions() throws IOException {- if (posIn.getFilePointer() == lastPosBlockFP) {- final int count = (int) (totalTermFreq % BLOCK_SIZE);- int payloadLength = 0;- int offsetLength = 0;- payloadByteUpto = 0;- for (int i = 0; i < count; i++) {- int code = posIn.readVInt();- if (indexHasPayloads) {- if ((code & 1) != 0) {- payloadLength = posIn.readVInt();- }- payloadLengthBuffer[i] = payloadLength;- posDeltaBuffer[i] = code >>> 1;- if (payloadLength != 0) {- if (payloadByteUpto + payloadLength > payloadBytes.length) {- payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payloadLength);- }- posIn.readBytes(payloadBytes, payloadByteUpto, payloadLength);- payloadByteUpto += payloadLength;- }- } else {- posDeltaBuffer[i] = code;- }-- if (indexHasOffsets) {- int deltaCode = posIn.readVInt();- if ((deltaCode & 1) != 0) {- offsetLength = posIn.readVInt();- }- offsetStartDeltaBuffer[i] = deltaCode >>> 1;- offsetLengthBuffer[i] = offsetLength;- }- }- payloadByteUpto = 0;- } else {- pforUtil.decode(posIn, posDeltaBuffer);-- if (indexHasPayloads && payIn != null) {- if (needsPayloads) {- pforUtil.decode(payIn, payloadLengthBuffer);- int numBytes = payIn.readVInt();-- if (numBytes > payloadBytes.length) {- payloadBytes = ArrayUtil.grow(payloadBytes, numBytes);- }- payIn.readBytes(payloadBytes, 0, numBytes);- } else {- // this works, because when writing a vint block we always force the first length to be- // written- pforUtil.skip(payIn); // skip over lengths- int numBytes = payIn.readVInt(); // read length of payloadBytes- payIn.seek(payIn.getFilePointer() + numBytes); // skip over payloadBytes- }- payloadByteUpto = 0;- }-- if (indexHasOffsets && payIn != null) {- if (needsOffsets) {- pforUtil.decode(payIn, offsetStartDeltaBuffer);- pforUtil.decode(payIn, offsetLengthBuffer);- } else {- // this works, because when writing a vint block we always force the first length to be- // written- pforUtil.skip(payIn); // skip over starts- pforUtil.skip(payIn); // skip over lengths- }- }- }- }-- @Override- public void advanceShallow(int target) throws IOException {- if (target > nextSkipDoc) {- // always plus one to fix the result, since skip position in Lucene84SkipReader- // is a little different from MultiLevelSkipListReader- final int newDocUpto = skipper.skipTo(target) + 1;-- if (newDocUpto > docUpto) {- // Skipper moved- assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;- docUpto = newDocUpto;- posDocUpTo = docUpto;-- // Force to read next block- docBufferUpto = BLOCK_SIZE;- accum = skipper.getDoc();- posPendingFP = skipper.getPosPointer();- payPendingFP = skipper.getPayPointer();- posPendingCount = skipper.getPosBufferUpto();- lastStartOffset = 0; // new document- payloadByteUpto = skipper.getPayloadByteUpto(); // actually, this is just lastSkipEntry- seekTo = skipper.getDocPointer(); // delay the seek- }- // next time we call advance, this is used to- // foresee whether skipper is necessary.- nextSkipDoc = skipper.getNextSkipDoc();- }- assert nextSkipDoc >= target;- }-- @Override- public Impacts getImpacts() throws IOException {- advanceShallow(doc);- return skipper.getImpacts();- }-- @Override- public int nextDoc() throws IOException {- return advance(doc + 1);- }-- @Override- public int advance(int target) throws IOException {- if (target > nextSkipDoc) {- advanceShallow(target);- }- if (docBufferUpto == BLOCK_SIZE) {- if (seekTo >= 0) {- docIn.seek(seekTo);- seekTo = -1;- isFreqsRead = true; // reset isFreqsRead- }- refillDocs();- }-- // Now scan:- long doc;- while (true) {- doc = docBuffer[docBufferUpto];- docBufferUpto++;- docUpto++;-- if (doc >= target) {- break;- }-- if (docBufferUpto == BLOCK_SIZE) {- return this.doc = NO_MORE_DOCS;- }- }- position = 0;- lastStartOffset = 0;-- return this.doc = (int) doc;- }-- // TODO: in theory we could avoid loading frq block- // when not needed, ie, use skip data to load how far to- // seek the pos pointer ... instead of having to load frq- // blocks only to sum up how many positions to skip- private void skipPositions() throws IOException {- // Skip positions now:- int toSkip = posPendingCount - (int) freqBuffer[docBufferUpto - 1];- // if (DEBUG) {- // System.out.println(" FPR.skipPositions: toSkip=" + toSkip);- // }-- final int leftInBlock = BLOCK_SIZE - posBufferUpto;- if (toSkip < leftInBlock) {- int end = posBufferUpto + toSkip;- while (posBufferUpto < end) {- if (indexHasPayloads) {- payloadByteUpto += payloadLengthBuffer[posBufferUpto];- }- posBufferUpto++;- }- } else {- toSkip -= leftInBlock;- while (toSkip >= BLOCK_SIZE) {- assert posIn.getFilePointer() != lastPosBlockFP;- pforUtil.skip(posIn);-- if (indexHasPayloads && payIn != null) {- // Skip payloadLength block:- pforUtil.skip(payIn);-- // Skip payloadBytes block:- int numBytes = payIn.readVInt();- payIn.seek(payIn.getFilePointer() + numBytes);- }-- if (indexHasOffsets && payIn != null) {- pforUtil.skip(payIn);- pforUtil.skip(payIn);- }- toSkip -= BLOCK_SIZE;- }- refillPositions();- payloadByteUpto = 0;- posBufferUpto = 0;- while (posBufferUpto < toSkip) {- if (indexHasPayloads) {- payloadByteUpto += payloadLengthBuffer[posBufferUpto];- }- posBufferUpto++;- }- }-- position = 0;- lastStartOffset = 0;- }-- @Override- public int nextPosition() throws IOException {- if (indexHasPos == false || needsPositions == false) {- return -1;- }-- if (isFreqsRead == false) {- pforUtil.decode(docIn, freqBuffer); // read freqBuffer for this docs block- isFreqsRead = true;- }- while (posDocUpTo < docUpto) { // bring posPendingCount upto the current doc- posPendingCount += freqBuffer[docBufferUpto - (docUpto - posDocUpTo)];- posDocUpTo++;- }-- assert posPendingCount > 0;-- if (posPendingFP != -1) {- posIn.seek(posPendingFP);- posPendingFP = -1;-- if (payPendingFP != -1 && payIn != null) {- payIn.seek(payPendingFP);- payPendingFP = -1;- }-- // Force buffer refill:- posBufferUpto = BLOCK_SIZE;- }-- if (posPendingCount > freqBuffer[docBufferUpto - 1]) {- skipPositions();- posPendingCount = (int) freqBuffer[docBufferUpto - 1];- }-- if (posBufferUpto == BLOCK_SIZE) {- refillPositions();- posBufferUpto = 0;- }- position += posDeltaBuffer[posBufferUpto];-- if (indexHasPayloads) {- payloadLength = (int) payloadLengthBuffer[posBufferUpto];- payload.bytes = payloadBytes;- payload.offset = payloadByteUpto;- payload.length = payloadLength;- payloadByteUpto += payloadLength;- }-- if (indexHasOffsets && needsOffsets) {- startOffset = lastStartOffset + (int) offsetStartDeltaBuffer[posBufferUpto];- endOffset = startOffset + (int) offsetLengthBuffer[posBufferUpto];- lastStartOffset = startOffset;- }-- posBufferUpto++;- posPendingCount--;- return position;- }-- @Override- public int startOffset() {- return startOffset;- }-- @Override- public int endOffset() {- return endOffset;- }-- @Override- public BytesRef getPayload() {- if (payloadLength == 0) {- return null;- } else {- return payload;- }- }-- @Override- public long cost() {- return docFreq;- }- }-- @Override- public void checkIntegrity() throws IOException {- if (docIn != null) {- CodecUtil.checksumEntireFile(docIn);- }- if (posIn != null) {- CodecUtil.checksumEntireFile(posIn);- }- if (payIn != null) {- CodecUtil.checksumEntireFile(payIn);- }- }-- @Override- public String toString() {- return getClass().getSimpleName()- + "(positions="- + (posIn != null)- + ",payloads="- + (payIn != null)- + ")";- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84ScoreSkipReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84ScoreSkipReader.javadeleted file mode 100644index 6a186afdc3e..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84ScoreSkipReader.java+++ /dev/null@@ -1,135 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene84;--import java.io.IOException;-import java.util.Arrays;-import org.apache.lucene.index.FreqAndNormBuffer;-import org.apache.lucene.index.Impacts;-import org.apache.lucene.store.ByteArrayDataInput;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.util.ArrayUtil;--final class Lucene84ScoreSkipReader extends Lucene84SkipReader {-- private final byte[][] impactData;- private final int[] impactDataLength;- private final ByteArrayDataInput badi = new ByteArrayDataInput();- private final Impacts impacts;- private int numLevels = 1;- private final FreqAndNormBuffer[] perLevelImpacts;-- public Lucene84ScoreSkipReader(- IndexInput skipStream,- int maxSkipLevels,- boolean hasPos,- boolean hasOffsets,- boolean hasPayloads) {- super(skipStream, maxSkipLevels, hasPos, hasOffsets, hasPayloads);- this.impactData = new byte[maxSkipLevels][];- Arrays.fill(impactData, new byte[0]);- this.impactDataLength = new int[maxSkipLevels];- this.perLevelImpacts = new FreqAndNormBuffer[maxSkipLevels];- for (int i = 0; i < perLevelImpacts.length; ++i) {- perLevelImpacts[i] = new FreqAndNormBuffer();- perLevelImpacts[i].add(Integer.MAX_VALUE, 1L);- }- impacts =- new Impacts() {-- @Override- public int numLevels() {- return numLevels;- }-- @Override- public int getDocIdUpTo(int level) {- return skipDoc[level];- }-- @Override- public FreqAndNormBuffer getImpacts(int level) {- assert level < numLevels;- if (impactDataLength[level] > 0) {- badi.reset(impactData[level], 0, impactDataLength[level]);- perLevelImpacts[level] = readImpacts(badi, perLevelImpacts[level]);- impactDataLength[level] = 0;- }- return perLevelImpacts[level];- }- };- }-- @Override- public int skipTo(int target) throws IOException {- int result = super.skipTo(target);- if (numberOfSkipLevels > 0) {- numLevels = numberOfSkipLevels;- } else {- // End of postings don't have skip data anymore, so we fill with dummy data- // like SlowImpactsEnum.- numLevels = 1;- perLevelImpacts[0].size = 1;- perLevelImpacts[0].freqs[0] = Integer.MAX_VALUE;- perLevelImpacts[0].norms[0] = 1L;- impactDataLength[0] = 0;- }- return result;- }-- Impacts getImpacts() {- return impacts;- }-- @Override- protected void readImpacts(int level, IndexInput skipStream) throws IOException {- int length = skipStream.readVInt();- if (impactData[level].length < length) {- impactData[level] = new byte[ArrayUtil.oversize(length, Byte.BYTES)];- }- skipStream.readBytes(impactData[level], 0, length);- impactDataLength[level] = length;- }-- static FreqAndNormBuffer readImpacts(ByteArrayDataInput in, FreqAndNormBuffer reuse) {- int maxNumImpacts = in.length(); // at most one impact per byte- reuse.growNoCopy(maxNumImpacts);-- int freq = 0;- long norm = 0;- int size = 0;- while (in.getPosition() < in.length()) {- int freqDelta = in.readVInt();- if ((freqDelta & 0x01) != 0) {- freq += 1 + (freqDelta >>> 1);- try {- norm += 1 + in.readZLong();- } catch (IOException e) {- throw new RuntimeException(e); // cannot happen on a BADI- }- } else {- freq += 1 + (freqDelta >>> 1);- norm++;- }- reuse.freqs[size] = freq;- reuse.norms[size] = norm;- size++;- }- reuse.size = size;- return reuse;- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84SkipReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84SkipReader.javadeleted file mode 100644index 78b49e28195..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84SkipReader.java+++ /dev/null@@ -1,206 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene84;--import java.io.IOException;-import java.util.Arrays;-import org.apache.lucene.codecs.MultiLevelSkipListReader;-import org.apache.lucene.store.IndexInput;--/**- * Implements the skip list reader for block postings format that stores positions and payloads.- *- * <p>Although this skipper uses MultiLevelSkipListReader as an interface, its definition of skip- * position will be a little different.- *- * <p>For example, when skipInterval = blockSize = 3, df = 2*skipInterval = 6,- *- * <pre>- * 0 1 2 3 4 5- * d d d d d d (posting list)- * ^ ^ (skip point in MultiLeveSkipWriter)- * ^ (skip point in Lucene84SkipWriter)- * </pre>- *- * <p>In this case, MultiLevelSkipListReader will use the last document as a skip point, while- * Lucene84SkipReader should assume no skip point will comes.- *- * <p>If we use the interface directly in Lucene84SkipReader, it may silly try to read another skip- * data after the only skip point is loaded.- *- * <p>To illustrate this, we can call skipTo(d[5]), since skip point d[3] has smaller docId, and- * numSkipped+blockSize== df, the MultiLevelSkipListReader will assume the skip list isn't exhausted- * yet, and try to load a non-existed skip point- *- * <p>Therefore, we'll trim df before passing it to the interface. see trim(int)- */-class Lucene84SkipReader extends MultiLevelSkipListReader {- private long[] docPointer;- private long[] posPointer;- private long[] payPointer;- private int[] posBufferUpto;- private int[] payloadByteUpto;-- private long lastPosPointer;- private long lastPayPointer;- private int lastPayloadByteUpto;- private long lastDocPointer;- private int lastPosBufferUpto;-- public Lucene84SkipReader(- IndexInput skipStream,- int maxSkipLevels,- boolean hasPos,- boolean hasOffsets,- boolean hasPayloads) {- super(skipStream, maxSkipLevels, ForUtil.BLOCK_SIZE, 8);- docPointer = new long[maxSkipLevels];- if (hasPos) {- posPointer = new long[maxSkipLevels];- posBufferUpto = new int[maxSkipLevels];- if (hasPayloads) {- payloadByteUpto = new int[maxSkipLevels];- } else {- payloadByteUpto = null;- }- if (hasOffsets || hasPayloads) {- payPointer = new long[maxSkipLevels];- } else {- payPointer = null;- }- } else {- posPointer = null;- }- }-- /**- * Trim original docFreq to tell skipReader read proper number of skip points.- *- * <p>Since our definition in Lucene84Skip* is a little different from MultiLevelSkip* This- * trimmed docFreq will prevent skipReader from: 1. silly reading a non-existed skip point after- * the last block boundary 2. moving into the vInt block- */- protected int trim(int df) {- return df % ForUtil.BLOCK_SIZE == 0 ? df - 1 : df;- }-- public void init(- long skipPointer, long docBasePointer, long posBasePointer, long payBasePointer, int df)- throws IOException {- super.init(skipPointer, trim(df));- lastDocPointer = docBasePointer;- lastPosPointer = posBasePointer;- lastPayPointer = payBasePointer;-- Arrays.fill(docPointer, docBasePointer);- if (posPointer != null) {- Arrays.fill(posPointer, posBasePointer);- if (payPointer != null) {- Arrays.fill(payPointer, payBasePointer);- }- } else {- assert posBasePointer == 0;- }- }-- /**- * Returns the doc pointer of the doc to which the last call of {@link- * MultiLevelSkipListReader#skipTo(int)} has skipped.- */- public long getDocPointer() {- return lastDocPointer;- }-- public long getPosPointer() {- return lastPosPointer;- }-- public int getPosBufferUpto() {- return lastPosBufferUpto;- }-- public long getPayPointer() {- return lastPayPointer;- }-- public int getPayloadByteUpto() {- return lastPayloadByteUpto;- }-- public int getNextSkipDoc() {- return skipDoc[0];- }-- @Override- protected void seekChild(int level) throws IOException {- super.seekChild(level);- docPointer[level] = lastDocPointer;- if (posPointer != null) {- posPointer[level] = lastPosPointer;- posBufferUpto[level] = lastPosBufferUpto;- if (payloadByteUpto != null) {- payloadByteUpto[level] = lastPayloadByteUpto;- }- if (payPointer != null) {- payPointer[level] = lastPayPointer;- }- }- }-- @Override- protected void setLastSkipData(int level) {- super.setLastSkipData(level);- lastDocPointer = docPointer[level];-- if (posPointer != null) {- lastPosPointer = posPointer[level];- lastPosBufferUpto = posBufferUpto[level];- if (payPointer != null) {- lastPayPointer = payPointer[level];- }- if (payloadByteUpto != null) {- lastPayloadByteUpto = payloadByteUpto[level];- }- }- }-- @Override- protected int readSkipData(int level, IndexInput skipStream) throws IOException {- int delta = skipStream.readVInt();- docPointer[level] += skipStream.readVLong();-- if (posPointer != null) {- posPointer[level] += skipStream.readVLong();- posBufferUpto[level] = skipStream.readVInt();-- if (payloadByteUpto != null) {- payloadByteUpto[level] = skipStream.readVInt();- }-- if (payPointer != null) {- payPointer[level] += skipStream.readVLong();- }- }- readImpacts(level, skipStream);- return delta;- }-- // The default impl skips impacts- protected void readImpacts(int level, IndexInput skipStream) throws IOException {- skipStream.skipBytes(skipStream.readVInt());- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/PForUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/PForUtil.javadeleted file mode 100644index 690bfa501f4..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/PForUtil.java+++ /dev/null@@ -1,123 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene84;--import java.io.IOException;-import java.util.Arrays;-import org.apache.lucene.store.DataInput;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.util.packed.PackedInts;--/** Utility class to encode sequences of 128 small positive integers. */-public final class PForUtil {-- static boolean allEqual(long[] l) {- for (int i = 1; i < ForUtil.BLOCK_SIZE; ++i) {- if (l[i] != l[0]) {- return false;- }- }- return true;- }-- private final ForUtil forUtil;-- PForUtil(ForUtil forUtil) {- this.forUtil = forUtil;- }-- /** Encode 128 integers from {@code longs} into {@code out}. */- void encode(long[] longs, DataOutput out) throws IOException {- // At most 7 exceptions- final long[] top8 = new long[8];- Arrays.fill(top8, -1L);- for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {- if (longs[i] > top8[0]) {- top8[0] = longs[i];- Arrays.sort(- top8); // For only 8 entries we just sort on every iteration instead of maintaining a PQ- }- }-- final int maxBitsRequired = PackedInts.bitsRequired(top8[7]);- // We store the patch on a byte, so we can't decrease the number of bits required by more than 8- final int patchedBitsRequired = Math.max(PackedInts.bitsRequired(top8[0]), maxBitsRequired - 8);- int numExceptions = 0;- final long maxUnpatchedValue = (1L << patchedBitsRequired) - 1;- for (int i = 1; i < 8; ++i) {- if (top8[i] > maxUnpatchedValue) {- numExceptions++;- }- }- final byte[] exceptions = new byte[numExceptions * 2];- if (numExceptions > 0) {- int exceptionCount = 0;- for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {- if (longs[i] > maxUnpatchedValue) {- exceptions[exceptionCount * 2] = (byte) i;- exceptions[exceptionCount * 2 + 1] = (byte) (longs[i] >>> patchedBitsRequired);- longs[i] &= maxUnpatchedValue;- exceptionCount++;- }- }- assert exceptionCount == numExceptions : exceptionCount + " " + numExceptions;- }-- if (allEqual(longs) && maxBitsRequired <= 8) {- for (int i = 0; i < numExceptions; ++i) {- exceptions[2 * i + 1] =- (byte) (Byte.toUnsignedLong(exceptions[2 * i + 1]) << patchedBitsRequired);- }- out.writeByte((byte) (numExceptions << 5));- out.writeVLong(longs[0]);- } else {- final int token = (numExceptions << 5) | patchedBitsRequired;- out.writeByte((byte) token);- forUtil.encode(longs, patchedBitsRequired, out);- }- out.writeBytes(exceptions, exceptions.length);- }-- /** Decode 128 integers into {@code longs}. */- void decode(DataInput in, long[] longs) throws IOException {- final int token = Byte.toUnsignedInt(in.readByte());- final int bitsPerValue = token & 0x1f;- final int numExceptions = token >>> 5;- if (bitsPerValue == 0) {- Arrays.fill(longs, 0, ForUtil.BLOCK_SIZE, in.readVLong());- } else {- forUtil.decode(bitsPerValue, in, longs);- }- for (int i = 0; i < numExceptions; ++i) {- longs[Byte.toUnsignedInt(in.readByte())] |=- Byte.toUnsignedLong(in.readByte()) << bitsPerValue;- }- }-- /** Skip 128 integers. */- void skip(DataInput in) throws IOException {- final int token = Byte.toUnsignedInt(in.readByte());- final int bitsPerValue = token & 0x1f;- final int numExceptions = token >>> 5;- if (bitsPerValue == 0) {- in.readVLong();- in.skipBytes((numExceptions << 1));- } else {- in.skipBytes(forUtil.numBytes(bitsPerValue) + (numExceptions << 1));- }- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/gen_ForUtil.py b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/gen_ForUtil.pydeleted file mode 100644index a4531203ea5..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/gen_ForUtil.py+++ /dev/null@@ -1,582 +0,0 @@-#! /usr/bin/env python--# Licensed to the Apache Software Foundation (ASF) under one or more-# contributor license agreements. See the NOTICE file distributed with-# this work for additional information regarding copyright ownership.-# The ASF licenses this file to You under the Apache License, Version 2.0-# (the "License"); you may not use this file except in compliance with-# the License. You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.---"""Code generation for ForUtil.java"""--MAX_SPECIALIZED_BITS_PER_VALUE = 24-OUTPUT_FILE = "ForUtil.java"-PRIMITIVE_SIZE = [8, 16, 32]-HEADER = """// This file has been automatically generated, DO NOT EDIT--/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene84;--import java.io.IOException;-import org.apache.lucene.store.DataInput;-import org.apache.lucene.store.DataOutput;--// Inspired from https://fulmicoton.com/posts/bitpacking/-// Encodes multiple integers in a long to get SIMD-like speedups.-// If bitsPerValue <= 8 then we pack 8 ints per long-// else if bitsPerValue <= 16 we pack 4 ints per long-// else we pack 2 ints per long-final class ForUtil {-- static final int BLOCK_SIZE = 128;- private static final int BLOCK_SIZE_LOG2 = 7;-- private static long expandMask32(long mask32) {- return mask32 | (mask32 << 32);- }-- private static long expandMask16(long mask16) {- return expandMask32(mask16 | (mask16 << 16));- }-- private static long expandMask8(long mask8) {- return expandMask16(mask8 | (mask8 << 8));- }-- private static long mask32(int bitsPerValue) {- return expandMask32((1L << bitsPerValue) - 1);- }-- private static long mask16(int bitsPerValue) {- return expandMask16((1L << bitsPerValue) - 1);- }-- private static long mask8(int bitsPerValue) {- return expandMask8((1L << bitsPerValue) - 1);- }-- private static void expand8(long[] arr) {- for (int i = 0; i < 16; ++i) {- long l = arr[i];- arr[i] = (l >>> 56) & 0xFFL;- arr[16 + i] = (l >>> 48) & 0xFFL;- arr[32 + i] = (l >>> 40) & 0xFFL;- arr[48 + i] = (l >>> 32) & 0xFFL;- arr[64 + i] = (l >>> 24) & 0xFFL;- arr[80 + i] = (l >>> 16) & 0xFFL;- arr[96 + i] = (l >>> 8) & 0xFFL;- arr[112 + i] = l & 0xFFL;- }- }-- private static void expand8To32(long[] arr) {- for (int i = 0; i < 16; ++i) {- long l = arr[i];- arr[i] = (l >>> 24) & 0x000000FF000000FFL;- arr[16 + i] = (l >>> 16) & 0x000000FF000000FFL;- arr[32 + i] = (l >>> 8) & 0x000000FF000000FFL;- arr[48 + i] = l & 0x000000FF000000FFL;- }- }-- private static void collapse8(long[] arr) {- for (int i = 0; i < 16; ++i) {- arr[i] =- (arr[i] << 56)- | (arr[16 + i] << 48)- | (arr[32 + i] << 40)- | (arr[48 + i] << 32)- | (arr[64 + i] << 24)- | (arr[80 + i] << 16)- | (arr[96 + i] << 8)- | arr[112 + i];- }- }-- private static void expand16(long[] arr) {- for (int i = 0; i < 32; ++i) {- long l = arr[i];- arr[i] = (l >>> 48) & 0xFFFFL;- arr[32 + i] = (l >>> 32) & 0xFFFFL;- arr[64 + i] = (l >>> 16) & 0xFFFFL;- arr[96 + i] = l & 0xFFFFL;- }- }-- private static void expand16To32(long[] arr) {- for (int i = 0; i < 32; ++i) {- long l = arr[i];- arr[i] = (l >>> 16) & 0x0000FFFF0000FFFFL;- arr[32 + i] = l & 0x0000FFFF0000FFFFL;- }- }-- private static void collapse16(long[] arr) {- for (int i = 0; i < 32; ++i) {- arr[i] = (arr[i] << 48) | (arr[32 + i] << 32) | (arr[64 + i] << 16) | arr[96 + i];- }- }-- private static void expand32(long[] arr) {- for (int i = 0; i < 64; ++i) {- long l = arr[i];- arr[i] = l >>> 32;- arr[64 + i] = l & 0xFFFFFFFFL;- }- }-- private static void collapse32(long[] arr) {- for (int i = 0; i < 64; ++i) {- arr[i] = (arr[i] << 32) | arr[64 + i];- }- }-- private static void prefixSum8(long[] arr, long base) {- expand8To32(arr);- prefixSum32(arr, base);- }-- private static void prefixSum16(long[] arr, long base) {- // We need to move to the next primitive size to avoid overflows- expand16To32(arr);- prefixSum32(arr, base);- }-- private static void prefixSum32(long[] arr, long base) {- arr[0] += base << 32;- innerPrefixSum32(arr);- expand32(arr);- final long l = arr[BLOCK_SIZE / 2 - 1];- for (int i = BLOCK_SIZE / 2; i < BLOCK_SIZE; ++i) {- arr[i] += l;- }- }-- // For some reason unrolling seems to help- private static void innerPrefixSum32(long[] arr) {- arr[1] += arr[0];- arr[2] += arr[1];- arr[3] += arr[2];- arr[4] += arr[3];- arr[5] += arr[4];- arr[6] += arr[5];- arr[7] += arr[6];- arr[8] += arr[7];- arr[9] += arr[8];- arr[10] += arr[9];- arr[11] += arr[10];- arr[12] += arr[11];- arr[13] += arr[12];- arr[14] += arr[13];- arr[15] += arr[14];- arr[16] += arr[15];- arr[17] += arr[16];- arr[18] += arr[17];- arr[19] += arr[18];- arr[20] += arr[19];- arr[21] += arr[20];- arr[22] += arr[21];- arr[23] += arr[22];- arr[24] += arr[23];- arr[25] += arr[24];- arr[26] += arr[25];- arr[27] += arr[26];- arr[28] += arr[27];- arr[29] += arr[28];- arr[30] += arr[29];- arr[31] += arr[30];- arr[32] += arr[31];- arr[33] += arr[32];- arr[34] += arr[33];- arr[35] += arr[34];- arr[36] += arr[35];- arr[37] += arr[36];- arr[38] += arr[37];- arr[39] += arr[38];- arr[40] += arr[39];- arr[41] += arr[40];- arr[42] += arr[41];- arr[43] += arr[42];- arr[44] += arr[43];- arr[45] += arr[44];- arr[46] += arr[45];- arr[47] += arr[46];- arr[48] += arr[47];- arr[49] += arr[48];- arr[50] += arr[49];- arr[51] += arr[50];- arr[52] += arr[51];- arr[53] += arr[52];- arr[54] += arr[53];- arr[55] += arr[54];- arr[56] += arr[55];- arr[57] += arr[56];- arr[58] += arr[57];- arr[59] += arr[58];- arr[60] += arr[59];- arr[61] += arr[60];- arr[62] += arr[61];- arr[63] += arr[62];- }-- private static void readLELongs(DataInput in, long[] dst, int offset, int length)- throws IOException {- in.readLongs(dst, offset, length);- for (int i = 0; i < length; ++i) {- dst[offset + i] = Long.reverseBytes(dst[offset + i]);- }- }-- private final long[] tmp = new long[BLOCK_SIZE / 2];-- /** Encode 128 integers from {@code longs} into {@code out}. */- void encode(long[] longs, int bitsPerValue, DataOutput out) throws IOException {- final int nextPrimitive;- final int numLongs;- if (bitsPerValue <= 8) {- nextPrimitive = 8;- numLongs = BLOCK_SIZE / 8;- collapse8(longs);- } else if (bitsPerValue <= 16) {- nextPrimitive = 16;- numLongs = BLOCK_SIZE / 4;- collapse16(longs);- } else {- nextPrimitive = 32;- numLongs = BLOCK_SIZE / 2;- collapse32(longs);- }-- final int numLongsPerShift = bitsPerValue * 2;- int idx = 0;- int shift = nextPrimitive - bitsPerValue;- for (int i = 0; i < numLongsPerShift; ++i) {- tmp[i] = longs[idx++] << shift;- }- for (shift = shift - bitsPerValue; shift >= 0; shift -= bitsPerValue) {- for (int i = 0; i < numLongsPerShift; ++i) {- tmp[i] |= longs[idx++] << shift;- }- }-- final int remainingBitsPerLong = shift + bitsPerValue;- final long maskRemainingBitsPerLong;- if (nextPrimitive == 8) {- maskRemainingBitsPerLong = MASKS8[remainingBitsPerLong];- } else if (nextPrimitive == 16) {- maskRemainingBitsPerLong = MASKS16[remainingBitsPerLong];- } else {- maskRemainingBitsPerLong = MASKS32[remainingBitsPerLong];- }-- int tmpIdx = 0;- int remainingBitsPerValue = bitsPerValue;- while (idx < numLongs) {- if (remainingBitsPerValue >= remainingBitsPerLong) {- remainingBitsPerValue -= remainingBitsPerLong;- tmp[tmpIdx++] |= (longs[idx] >>> remainingBitsPerValue) & maskRemainingBitsPerLong;- if (remainingBitsPerValue == 0) {- idx++;- remainingBitsPerValue = bitsPerValue;- }- } else {- final long mask1, mask2;- if (nextPrimitive == 8) {- mask1 = MASKS8[remainingBitsPerValue];- mask2 = MASKS8[remainingBitsPerLong - remainingBitsPerValue];- } else if (nextPrimitive == 16) {- mask1 = MASKS16[remainingBitsPerValue];- mask2 = MASKS16[remainingBitsPerLong - remainingBitsPerValue];- } else {- mask1 = MASKS32[remainingBitsPerValue];- mask2 = MASKS32[remainingBitsPerLong - remainingBitsPerValue];- }- tmp[tmpIdx] |= (longs[idx++] & mask1) << (remainingBitsPerLong - remainingBitsPerValue);- remainingBitsPerValue = bitsPerValue - remainingBitsPerLong + remainingBitsPerValue;- tmp[tmpIdx++] |= (longs[idx] >>> remainingBitsPerValue) & mask2;- }- }-- for (int i = 0; i < numLongsPerShift; ++i) {- // Java longs are big endian and we want to read little endian longs, so we need to reverse- // bytes- long l = Long.reverseBytes(tmp[i]);- out.writeLong(l);- }- }-- /** Number of bytes required to encode 128 integers of {@code bitsPerValue} bits per value. */- int numBytes(int bitsPerValue) throws IOException {- return bitsPerValue << (BLOCK_SIZE_LOG2 - 3);- }-- private static void decodeSlow(int bitsPerValue, DataInput in, long[] tmp, long[] longs)- throws IOException {- final int numLongs = bitsPerValue << 1;- readLELongs(in, tmp, 0, numLongs);- final long mask = MASKS32[bitsPerValue];- int longsIdx = 0;- int shift = 32 - bitsPerValue;- for (; shift >= 0; shift -= bitsPerValue) {- shiftLongs(tmp, numLongs, longs, longsIdx, shift, mask);- longsIdx += numLongs;- }- final int remainingBitsPerLong = shift + bitsPerValue;- final long mask32RemainingBitsPerLong = MASKS32[remainingBitsPerLong];- int tmpIdx = 0;- int remainingBits = remainingBitsPerLong;- for (; longsIdx < BLOCK_SIZE / 2; ++longsIdx) {- int b = bitsPerValue - remainingBits;- long l = (tmp[tmpIdx++] & MASKS32[remainingBits]) << b;- while (b >= remainingBitsPerLong) {- b -= remainingBitsPerLong;- l |= (tmp[tmpIdx++] & mask32RemainingBitsPerLong) << b;- }- if (b > 0) {- l |= (tmp[tmpIdx] >>> (remainingBitsPerLong - b)) & MASKS32[b];- remainingBits = remainingBitsPerLong - b;- } else {- remainingBits = remainingBitsPerLong;- }- longs[longsIdx] = l;- }- }-- /**- * The pattern that this shiftLongs method applies is recognized by the C2 compiler, which- * generates SIMD instructions for it in order to shift multiple longs at once.- */- private static void shiftLongs(long[] a, int count, long[] b, int bi, int shift, long mask) {- for (int i = 0; i < count; ++i) {- b[bi + i] = (a[i] >>> shift) & mask;- }- }--"""---def writeRemainderWithSIMDOptimize(- bpv, next_primitive, remaining_bits_per_long, o, num_values, f-):- iteration = 1- num_longs = bpv * num_values / remaining_bits_per_long- while num_longs % 2 == 0 and num_values % 2 == 0:- num_longs /= 2- num_values /= 2- iteration *= 2-- f.write(- " shiftLongs(tmp, %d, tmp, 0, 0, MASK%d_%d);\n"- % (iteration * num_longs, next_primitive, remaining_bits_per_long)- )- f.write(- " for (int iter = 0, tmpIdx = 0, longsIdx = %d; iter < %d; ++iter, tmpIdx += %d, longsIdx += %d) {\n"- % (o, iteration, num_longs, num_values)- )- tmp_idx = 0- b = bpv- b -= remaining_bits_per_long- f.write(" long l0 = tmp[tmpIdx + %d] << %d;\n" % (tmp_idx, b))- tmp_idx += 1- while b >= remaining_bits_per_long:- b -= remaining_bits_per_long- f.write(" l0 |= tmp[tmpIdx + %d] << %d;\n" % (tmp_idx, b))- tmp_idx += 1- f.write(" longs[longsIdx + 0] = l0;\n")- f.write(" }\n")---def writeRemainder(bpv, next_primitive, remaining_bits_per_long, o, num_values, f):- iteration = 1- num_longs = bpv * num_values / remaining_bits_per_long- while num_longs % 2 == 0 and num_values % 2 == 0:- num_longs /= 2- num_values /= 2- iteration *= 2- f.write(- " for (int iter = 0, tmpIdx = 0, longsIdx = %d; iter < %d; ++iter, tmpIdx += %d, longsIdx += %d) {\n"- % (o, iteration, num_longs, num_values)- )- i = 0- remaining_bits = 0- tmp_idx = 0- for i in range(int(num_values)):- b = bpv- if remaining_bits == 0:- b -= remaining_bits_per_long- f.write(- " long l%d = (tmp[tmpIdx + %d] & MASK%d_%d) << %d;\n"- % (i, tmp_idx, next_primitive, remaining_bits_per_long, b)- )- else:- b -= remaining_bits- f.write(- " long l%d = (tmp[tmpIdx + %d] & MASK%d_%d) << %d;\n"- % (i, tmp_idx, next_primitive, remaining_bits, b)- )- tmp_idx += 1- while b >= remaining_bits_per_long:- b -= remaining_bits_per_long- f.write(- " l%d |= (tmp[tmpIdx + %d] & MASK%d_%d) << %d;\n"- % (i, tmp_idx, next_primitive, remaining_bits_per_long, b)- )- tmp_idx += 1- if b > 0:- f.write(- " l%d |= (tmp[tmpIdx + %d] >>> %d) & MASK%d_%d;\n"- % (i, tmp_idx, remaining_bits_per_long - b, next_primitive, b)- )- remaining_bits = remaining_bits_per_long - b- f.write(" longs[longsIdx + %d] = l%d;\n" % (i, i))- f.write(" }\n")---def writeDecode(bpv, f):- next_primitive = 32- if bpv <= 8:- next_primitive = 8- elif bpv <= 16:- next_primitive = 16- f.write(- " private static void decode%d(DataInput in, long[] tmp, long[] longs) throws IOException {\n"- % bpv- )- num_values_per_long = 64 / next_primitive- if bpv == next_primitive:- f.write(" readLELongs(in, longs, 0, %d);\n" % (bpv * 2))- else:- f.write(" readLELongs(in, tmp, 0, %d);\n" % (bpv * 2))- shift = next_primitive - bpv- o = 0- while shift >= 0:- f.write(- " shiftLongs(tmp, %d, longs, %d, %d, MASK%d_%d);\n"- % (bpv * 2, o, shift, next_primitive, bpv)- )- o += bpv * 2- shift -= bpv- if shift + bpv > 0:- if bpv % (next_primitive % bpv) == 0:- writeRemainderWithSIMDOptimize(- bpv,- next_primitive,- shift + bpv,- o,- 128 / num_values_per_long - o,- f,- )- else:- writeRemainder(- bpv,- next_primitive,- shift + bpv,- o,- 128 / num_values_per_long - o,- f,- )- f.write(" }\n")- f.write("\n")---if __name__ == "__main__":- f = open(OUTPUT_FILE, "w")- f.write(HEADER)- for primitive_size in PRIMITIVE_SIZE:- f.write(- " private static final long[] MASKS%d = new long[%d];\n"- % (primitive_size, primitive_size)- )- f.write("\n")- f.write(" static {\n")- for primitive_size in PRIMITIVE_SIZE:- f.write(" for (int i = 0; i < %d; ++i) {\n" % primitive_size)- f.write(" MASKS%d[i] = mask%d(i);\n" % (primitive_size, primitive_size))- f.write(" }\n")- f.write(" }\n")- f.write(- " // mark values in array as final longs to avoid the cost of reading array, arrays should only be\n"- )- f.write(" // used when the idx is a variable\n")- for primitive_size in PRIMITIVE_SIZE:- for bpv in range(1, min(MAX_SPECIALIZED_BITS_PER_VALUE + 1, primitive_size)):- if bpv * 2 != primitive_size or primitive_size == 8:- f.write(- " private static final long MASK%d_%d = MASKS%d[%d];\n"- % (primitive_size, bpv, primitive_size, bpv)- )-- f.write("""- /** Decode 128 integers into {@code longs}. */- void decode(int bitsPerValue, DataInput in, long[] longs) throws IOException {- switch (bitsPerValue) {-""")- for bpv in range(1, MAX_SPECIALIZED_BITS_PER_VALUE + 1):- next_primitive = 32- if bpv <= 8:- next_primitive = 8- elif bpv <= 16:- next_primitive = 16- f.write(" case %d:\n" % bpv)- f.write(" decode%d(in, tmp, longs);\n" % bpv)- f.write(" expand%d(longs);\n" % next_primitive)- f.write(" break;\n")- f.write(" default:\n")- f.write(" decodeSlow(bitsPerValue, in, tmp, longs);\n")- f.write(" expand32(longs);\n")- f.write(" break;\n")- f.write(" }\n")- f.write(" }\n")-- f.write("""- /** Delta-decode 128 integers into {@code longs}. */- void decodeAndPrefixSum(int bitsPerValue, DataInput in, long base, long[] longs)- throws IOException {- switch (bitsPerValue) {-""")- for bpv in range(1, MAX_SPECIALIZED_BITS_PER_VALUE + 1):- next_primitive = 32- if bpv <= 8:- next_primitive = 8- elif bpv <= 16:- next_primitive = 16- f.write(" case %d:\n" % bpv)- f.write(" decode%d(in, tmp, longs);\n" % bpv)- f.write(" prefixSum%d(longs, base);\n" % next_primitive)- f.write(" break;\n")- f.write(" default:\n")- f.write(" decodeSlow(bitsPerValue, in, tmp, longs);\n")- f.write(" prefixSum32(longs, base);\n")- f.write(" break;\n")- f.write(" }\n")- f.write(" }\n")-- f.write("\n")- for i in range(1, MAX_SPECIALIZED_BITS_PER_VALUE + 1):- writeDecode(i, f)-- f.write("}\n")diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/package-info.javadeleted file mode 100644index b527657a699..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/package-info.java+++ /dev/null@@ -1,19 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--/** Lucene 8.4 file format. */-package org.apache.lucene.backward_codecs.lucene84;diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86Codec.javadeleted file mode 100644index 2c56e7676c6..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86Codec.java+++ /dev/null@@ -1,178 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--package org.apache.lucene.backward_codecs.lucene86;--import java.util.Objects;-import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;-import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;-import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat;-import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat;-import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;-import org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat;-import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat;-import org.apache.lucene.codecs.Codec;-import org.apache.lucene.codecs.CompoundFormat;-import org.apache.lucene.codecs.DocValuesFormat;-import org.apache.lucene.codecs.FieldInfosFormat;-import org.apache.lucene.codecs.FilterCodec;-import org.apache.lucene.codecs.KnnVectorsFormat;-import org.apache.lucene.codecs.LiveDocsFormat;-import org.apache.lucene.codecs.NormsFormat;-import org.apache.lucene.codecs.PointsFormat;-import org.apache.lucene.codecs.PostingsFormat;-import org.apache.lucene.codecs.SegmentInfoFormat;-import org.apache.lucene.codecs.StoredFieldsFormat;-import org.apache.lucene.codecs.TermVectorsFormat;-import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;-import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;--/**- * Implements the Lucene 8.6 index format, with configurable per-field postings and docvalues- * formats.- *- * <p>If you want to reuse functionality of this codec in another codec, extend {@link FilterCodec}.- *- * @lucene.experimental- */-public class Lucene86Codec extends Codec {- private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat();- private final FieldInfosFormat fieldInfosFormat = new Lucene60FieldInfosFormat();- private final SegmentInfoFormat segmentInfosFormat = new Lucene86SegmentInfoFormat();- private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();- private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();- private final PointsFormat pointsFormat = new Lucene86PointsFormat();- private final PostingsFormat defaultFormat;-- private final PostingsFormat postingsFormat =- new PerFieldPostingsFormat() {- @Override- public PostingsFormat getPostingsFormatForField(String field) {- return Lucene86Codec.this.getPostingsFormatForField(field);- }- };-- private final DocValuesFormat docValuesFormat =- new PerFieldDocValuesFormat() {- @Override- public DocValuesFormat getDocValuesFormatForField(String field) {- return Lucene86Codec.this.getDocValuesFormatForField(field);- }- };-- private final StoredFieldsFormat storedFieldsFormat;-- /** Instantiates a new codec. */- public Lucene86Codec() {- this(Lucene50StoredFieldsFormat.Mode.BEST_SPEED);- }-- /**- * Instantiates a new codec, specifying the stored fields compression mode to use.- *- * @param mode stored fields compression mode to use for newly flushed/merged segments.- */- public Lucene86Codec(Lucene50StoredFieldsFormat.Mode mode) {- super("Lucene86");- this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Objects.requireNonNull(mode));- this.defaultFormat = new Lucene84PostingsFormat();- }-- @Override- public StoredFieldsFormat storedFieldsFormat() {- return storedFieldsFormat;- }-- @Override- public TermVectorsFormat termVectorsFormat() {- return vectorsFormat;- }-- @Override- public PostingsFormat postingsFormat() {- return postingsFormat;- }-- @Override- public final FieldInfosFormat fieldInfosFormat() {- return fieldInfosFormat;- }-- @Override- public SegmentInfoFormat segmentInfoFormat() {- return segmentInfosFormat;- }-- @Override- public final LiveDocsFormat liveDocsFormat() {- return liveDocsFormat;- }-- @Override- public CompoundFormat compoundFormat() {- return compoundFormat;- }-- @Override- public PointsFormat pointsFormat() {- return pointsFormat;- }-- @Override- public final KnnVectorsFormat knnVectorsFormat() {- return KnnVectorsFormat.EMPTY;- }-- /**- * Returns the postings format that should be used for writing new segments of <code>field</code>.- *- * <p>The default implementation always returns "Lucene84".- *- * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:- * future version of Lucene are only guaranteed to be able to read the default implementation.- */- public PostingsFormat getPostingsFormatForField(String field) {- return defaultFormat;- }-- /**- * Returns the docvalues format that should be used for writing new segments of <code>field</code>- * .- *- * <p>The default implementation always returns "Lucene80".- *- * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:- * future version of Lucene are only guaranteed to be able to read the default implementation.- */- public DocValuesFormat getDocValuesFormatForField(String field) {- return defaultDVFormat;- }-- @Override- public final DocValuesFormat docValuesFormat() {- return docValuesFormat;- }-- private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene80");-- private final NormsFormat normsFormat = new Lucene80NormsFormat();-- @Override- public NormsFormat normsFormat() {- return normsFormat;- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86PointsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86PointsFormat.javadeleted file mode 100644index b8c4d8cd118..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86PointsFormat.java+++ /dev/null@@ -1,76 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene86;--import java.io.IOException;-import org.apache.lucene.codecs.PointsFormat;-import org.apache.lucene.codecs.PointsReader;-import org.apache.lucene.codecs.PointsWriter;-import org.apache.lucene.index.SegmentReadState;-import org.apache.lucene.index.SegmentWriteState;--/**- * Lucene 8.6 point format, which encodes dimensional values in a block KD-tree structure for fast- * 1D range and N dimensional shape intersection filtering. See <a- * href="https://www.cs.duke.edu/~pankaj/publications/papers/bkd-sstd.pdf">this paper</a> for- * details.- *- * <p>Data is stored across three files- *- * <ul>- * <li>A .kdm file that records metadata about the fields, such as numbers of dimensions or- * numbers of bytes per dimension.- * <li>A .kdi file that stores inner nodes of the tree.- * <li>A .kdd file that stores leaf nodes, where most of the data lives.- * </ul>- *- * See <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=173081898">this- * wiki</a> for detailed data structures of the three files.- *- * @lucene.experimental- */-public class Lucene86PointsFormat extends PointsFormat {-- static final String DATA_CODEC_NAME = "Lucene86PointsFormatData";- static final String INDEX_CODEC_NAME = "Lucene86PointsFormatIndex";- static final String META_CODEC_NAME = "Lucene86PointsFormatMeta";-- /** Filename extension for the leaf blocks */- public static final String DATA_EXTENSION = "kdd";-- /** Filename extension for the index per field */- public static final String INDEX_EXTENSION = "kdi";-- /** Filename extension for the meta per field */- public static final String META_EXTENSION = "kdm";-- static final int VERSION_START = 0;- static final int VERSION_CURRENT = VERSION_START;-- /** Sole constructor */- public Lucene86PointsFormat() {}-- @Override- public PointsWriter fieldsWriter(SegmentWriteState state) throws IOException {- throw new UnsupportedOperationException("Old codecs may only be used for reading");- }-- @Override- public PointsReader fieldsReader(SegmentReadState state) throws IOException {- return new Lucene86PointsReader(state);- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86PointsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86PointsReader.javadeleted file mode 100644index efe6f5dccfe..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86PointsReader.java+++ /dev/null@@ -1,157 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene86;--import java.io.IOException;-import java.util.HashMap;-import java.util.Map;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.PointsReader;-import org.apache.lucene.index.CorruptIndexException;-import org.apache.lucene.index.FieldInfo;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.PointValues;-import org.apache.lucene.index.SegmentReadState;-import org.apache.lucene.store.ChecksumIndexInput;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.util.IOUtils;-import org.apache.lucene.util.bkd.BKDReader;--/** Reads point values previously written with Lucene86PointsWriter */-public class Lucene86PointsReader extends PointsReader {- final IndexInput indexIn, dataIn;- final SegmentReadState readState;- final Map<Integer, PointValues> readers = new HashMap<>();-- /** Sole constructor */- public Lucene86PointsReader(SegmentReadState readState) throws IOException {- this.readState = readState;-- String metaFileName =- IndexFileNames.segmentFileName(- readState.segmentInfo.name,- readState.segmentSuffix,- Lucene86PointsFormat.META_EXTENSION);- String indexFileName =- IndexFileNames.segmentFileName(- readState.segmentInfo.name,- readState.segmentSuffix,- Lucene86PointsFormat.INDEX_EXTENSION);- String dataFileName =- IndexFileNames.segmentFileName(- readState.segmentInfo.name,- readState.segmentSuffix,- Lucene86PointsFormat.DATA_EXTENSION);-- boolean success = false;- try {- indexIn =- EndiannessReverserUtil.openInput(readState.directory, indexFileName, readState.context);- CodecUtil.checkIndexHeader(- indexIn,- Lucene86PointsFormat.INDEX_CODEC_NAME,- Lucene86PointsFormat.VERSION_START,- Lucene86PointsFormat.VERSION_CURRENT,- readState.segmentInfo.getId(),- readState.segmentSuffix);-- dataIn =- EndiannessReverserUtil.openInput(readState.directory, dataFileName, readState.context);- CodecUtil.checkIndexHeader(- dataIn,- Lucene86PointsFormat.DATA_CODEC_NAME,- Lucene86PointsFormat.VERSION_START,- Lucene86PointsFormat.VERSION_CURRENT,- readState.segmentInfo.getId(),- readState.segmentSuffix);-- long indexLength = -1, dataLength = -1;- try (ChecksumIndexInput metaIn =- EndiannessReverserUtil.openChecksumInput(- readState.directory, metaFileName, readState.context)) {- Throwable priorE = null;- try {- CodecUtil.checkIndexHeader(- metaIn,- Lucene86PointsFormat.META_CODEC_NAME,- Lucene86PointsFormat.VERSION_START,- Lucene86PointsFormat.VERSION_CURRENT,- readState.segmentInfo.getId(),- readState.segmentSuffix);-- while (true) {- int fieldNumber = metaIn.readInt();- if (fieldNumber == -1) {- break;- } else if (fieldNumber < 0) {- throw new CorruptIndexException("Illegal field number: " + fieldNumber, metaIn);- }- PointValues reader = new BKDReader(metaIn, indexIn, dataIn);- readers.put(fieldNumber, reader);- }- indexLength = metaIn.readLong();- dataLength = metaIn.readLong();- } catch (Throwable t) {- priorE = t;- } finally {- CodecUtil.checkFooter(metaIn, priorE);- }- }- // At this point, checksums of the meta file have been validated so we- // know that indexLength and dataLength are very likely correct.- CodecUtil.retrieveChecksum(indexIn, indexLength);- CodecUtil.retrieveChecksum(dataIn, dataLength);- success = true;- } finally {- if (success == false) {- IOUtils.closeWhileHandlingException(this);- }- }- }-- /**- * Returns the underlying {@link PointValues}.- *- * @lucene.internal- */- @Override- public PointValues getValues(String fieldName) {- FieldInfo fieldInfo = readState.fieldInfos.fieldInfo(fieldName);- if (fieldInfo == null) {- throw new IllegalArgumentException("field=\"" + fieldName + "\" is unrecognized");- }- if (fieldInfo.getPointDimensionCount() == 0) {- throw new IllegalArgumentException("field=\"" + fieldName + "\" did not index point values");- }-- return readers.get(fieldInfo.number);- }-- @Override- public void checkIntegrity() throws IOException {- CodecUtil.checksumEntireFile(indexIn);- CodecUtil.checksumEntireFile(dataIn);- }-- @Override- public void close() throws IOException {- IOUtils.close(indexIn, dataIn);- // Free up heap:- readers.clear();- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86SegmentInfoFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86SegmentInfoFormat.javadeleted file mode 100644index b7b660ff4f5..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86SegmentInfoFormat.java+++ /dev/null@@ -1,181 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--package org.apache.lucene.backward_codecs.lucene86;--import java.io.IOException;-import java.util.Map;-import java.util.Set;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.SegmentInfoFormat;-import org.apache.lucene.index.CorruptIndexException;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.IndexWriter;-import org.apache.lucene.index.SegmentInfo;-import org.apache.lucene.index.SegmentInfos;-import org.apache.lucene.index.SortFieldProvider;-import org.apache.lucene.search.Sort;-import org.apache.lucene.search.SortField;-import org.apache.lucene.store.ChecksumIndexInput;-import org.apache.lucene.store.DataInput;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.util.Version;--/**- * Lucene 8.6 Segment info format.- *- * <p>Files:- *- * <ul>- * <li><code>.si</code>: Header, SegVersion, SegSize, IsCompoundFile, Diagnostics, Files,- * Attributes, IndexSort, Footer- * </ul>- *- * Data types:- *- * <ul>- * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}- * <li>SegSize --> {@link DataOutput#writeInt Int32}- * <li>SegVersion --> {@link DataOutput#writeString String}- * <li>SegMinVersion --> {@link DataOutput#writeString String}- * <li>Files --> {@link DataOutput#writeSetOfStrings Set<String>}- * <li>Diagnostics,Attributes --> {@link DataOutput#writeMapOfStrings Map<String,String>}- * <li>IsCompoundFile --> {@link DataOutput#writeByte Int8}- * <li>IndexSort --> {@link DataOutput#writeVInt Int32} count, followed by {@code count}- * SortField- * <li>SortField --> {@link DataOutput#writeString String} sort class, followed by a per-sort- * bytestream (see {@link SortFieldProvider#readSortField(DataInput)})- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}- * </ul>- *- * Field Descriptions:- *- * <ul>- * <li>SegVersion is the code version that created the segment.- * <li>SegMinVersion is the minimum code version that contributed documents to the segment.- * <li>SegSize is the number of documents contained in the segment index.- * <li>IsCompoundFile records whether the segment is written as a compound file or not. If this is- * -1, the segment is not a compound file. If it is 1, the segment is a compound file.- * <li>The Diagnostics Map is privately written by {@link IndexWriter}, as a debugging aid, for- * each segment it creates. It includes metadata like the current Lucene version, OS, Java- * version, why the segment was created (merge, flush, addIndexes), etc.- * <li>Files is a list of files referred to by this segment.- * </ul>- *- * @see SegmentInfos- * @lucene.experimental- */-public class Lucene86SegmentInfoFormat extends SegmentInfoFormat {-- /** File extension used to store {@link SegmentInfo}. */- public static final String SI_EXTENSION = "si";-- static final String CODEC_NAME = "Lucene86SegmentInfo";- static final int VERSION_START = 0;- static final int VERSION_CURRENT = VERSION_START;-- /** Sole constructor. */- public Lucene86SegmentInfoFormat() {}-- @Override- public SegmentInfo read(Directory dir, String segment, byte[] segmentID, IOContext context)- throws IOException {- final String fileName = IndexFileNames.segmentFileName(segment, "", SI_EXTENSION);- try (ChecksumIndexInput input =- EndiannessReverserUtil.openChecksumInput(dir, fileName, context)) {- Throwable priorE = null;- SegmentInfo si = null;- try {- CodecUtil.checkIndexHeader(- input, CODEC_NAME, VERSION_START, VERSION_CURRENT, segmentID, "");- si = parseSegmentInfo(dir, input, segment, segmentID);- } catch (Throwable exception) {- priorE = exception;- } finally {- CodecUtil.checkFooter(input, priorE);- }- return si;- }- }-- private SegmentInfo parseSegmentInfo(- Directory dir, DataInput input, String segment, byte[] segmentID) throws IOException {- final Version version = Version.fromBits(input.readInt(), input.readInt(), input.readInt());- byte hasMinVersion = input.readByte();- final Version minVersion;- switch (hasMinVersion) {- case 0:- minVersion = null;- break;- case 1:- minVersion = Version.fromBits(input.readInt(), input.readInt(), input.readInt());- break;- default:- throw new CorruptIndexException("Illegal boolean value " + hasMinVersion, input);- }-- final int docCount = input.readInt();- if (docCount < 0) {- throw new CorruptIndexException("invalid docCount: " + docCount, input);- }- final boolean isCompoundFile = input.readByte() == SegmentInfo.YES;-- final Map<String, String> diagnostics = input.readMapOfStrings();- final Set<String> files = input.readSetOfStrings();- final Map<String, String> attributes = input.readMapOfStrings();-- int numSortFields = input.readVInt();- Sort indexSort;- if (numSortFields > 0) {- SortField[] sortFields = new SortField[numSortFields];- for (int i = 0; i < numSortFields; i++) {- String name = input.readString();- sortFields[i] = SortFieldProvider.forName(name).readSortField(input);- }- indexSort = new Sort(sortFields);- } else if (numSortFields < 0) {- throw new CorruptIndexException("invalid index sort field count: " + numSortFields, input);- } else {- indexSort = null;- }-- SegmentInfo si =- new SegmentInfo(- dir,- version,- minVersion,- segment,- docCount,- isCompoundFile,- false,- null,- diagnostics,- segmentID,- attributes,- indexSort);- si.setFiles(files);- return si;- }-- @Override- public void write(Directory dir, SegmentInfo si, IOContext ioContext) throws IOException {- throw new UnsupportedOperationException("Old formats can't be used for writing");- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/package-info.javadeleted file mode 100644index c516fa3ce69..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/package-info.java+++ /dev/null@@ -1,19 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--/** Lucene 8.6 file format. */-package org.apache.lucene.backward_codecs.lucene86;diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/DeflateWithPresetDictCompressionMode.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/DeflateWithPresetDictCompressionMode.javadeleted file mode 100644index 8ff41e64a05..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/DeflateWithPresetDictCompressionMode.java+++ /dev/null@@ -1,228 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene87;--import java.io.IOException;-import java.util.zip.DataFormatException;-import java.util.zip.Deflater;-import java.util.zip.Inflater;-import org.apache.lucene.backward_codecs.compressing.CompressionMode;-import org.apache.lucene.backward_codecs.compressing.Compressor;-import org.apache.lucene.backward_codecs.compressing.Decompressor;-import org.apache.lucene.index.CorruptIndexException;-import org.apache.lucene.store.DataInput;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.BytesRef;--/**- * A compression mode that trades speed for compression ratio. Although compression and- * decompression might be slow, this compression mode should provide a good compression ratio. This- * mode might be interesting if/when your index size is much bigger than your OS cache.- *- * @lucene.internal- */-public final class DeflateWithPresetDictCompressionMode extends CompressionMode {-- // Shoot for 10 sub blocks- private static final int NUM_SUB_BLOCKS = 10;- // And a dictionary whose size is about 6x smaller than sub blocks- private static final int DICT_SIZE_FACTOR = 6;-- /** Sole constructor. */- public DeflateWithPresetDictCompressionMode() {}-- @Override- public Compressor newCompressor() {- // notes:- // 3 is the highest level that doesn't have lazy match evaluation- // 6 is the default, higher than that is just a waste of cpu- return new DeflateWithPresetDictCompressor(6);- }-- @Override- public Decompressor newDecompressor() {- return new DeflateWithPresetDictDecompressor();- }-- @Override- public String toString() {- return "BEST_COMPRESSION";- }-- private static final class DeflateWithPresetDictDecompressor extends Decompressor {-- byte[] compressed;-- DeflateWithPresetDictDecompressor() {- compressed = new byte[0];- }-- private void doDecompress(DataInput in, Inflater decompressor, BytesRef bytes)- throws IOException {- final int compressedLength = in.readVInt();- if (compressedLength == 0) {- return;- }- // pad with extra "dummy byte": see javadocs for using Inflater(true)- // we do it for compliance, but it's unnecessary for years in zlib.- final int paddedLength = compressedLength + 1;- compressed = ArrayUtil.grow(compressed, paddedLength);- in.readBytes(compressed, 0, compressedLength);- compressed[compressedLength] = 0; // explicitly set dummy byte to 0-- // extra "dummy byte"- decompressor.setInput(compressed, 0, paddedLength);- try {- bytes.length +=- decompressor.inflate(bytes.bytes, bytes.length, bytes.bytes.length - bytes.length);- } catch (DataFormatException e) {- throw new IOException(e);- }- if (decompressor.finished() == false) {- throw new CorruptIndexException(- "Invalid decoder state: needsInput="- + decompressor.needsInput()- + ", needsDict="- + decompressor.needsDictionary(),- in);- }- }-- @Override- public void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes)- throws IOException {- assert offset + length <= originalLength;- if (length == 0) {- bytes.length = 0;- return;- }- final int dictLength = in.readVInt();- final int blockLength = in.readVInt();- bytes.bytes = ArrayUtil.grow(bytes.bytes, dictLength);- bytes.offset = bytes.length = 0;-- final Inflater decompressor = new Inflater(true);- try {- // Read the dictionary- doDecompress(in, decompressor, bytes);- if (dictLength != bytes.length) {- throw new CorruptIndexException("Unexpected dict length", in);- }-- int offsetInBlock = dictLength;- int offsetInBytesRef = offset;-- // Skip unneeded blocks- while (offsetInBlock + blockLength < offset) {- final int compressedLength = in.readVInt();- in.skipBytes(compressedLength);- offsetInBlock += blockLength;- offsetInBytesRef -= blockLength;- }-- // Read blocks that intersect with the interval we need- while (offsetInBlock < offset + length) {- bytes.bytes = ArrayUtil.grow(bytes.bytes, bytes.length + blockLength);- decompressor.reset();- decompressor.setDictionary(bytes.bytes, 0, dictLength);- doDecompress(in, decompressor, bytes);- offsetInBlock += blockLength;- }-- bytes.offset = offsetInBytesRef;- bytes.length = length;- assert bytes.isValid();- } finally {- decompressor.end();- }- }-- @Override- public Decompressor clone() {- return new DeflateWithPresetDictDecompressor();- }- }-- private static class DeflateWithPresetDictCompressor extends Compressor {-- final Deflater compressor;- byte[] compressed;- boolean closed;-- DeflateWithPresetDictCompressor(int level) {- compressor = new Deflater(level, true);- compressed = new byte[64];- }-- private void doCompress(byte[] bytes, int off, int len, DataOutput out) throws IOException {- if (len == 0) {- out.writeVInt(0);- return;- }- compressor.setInput(bytes, off, len);- compressor.finish();- if (compressor.needsInput()) {- throw new IllegalStateException();- }-- int totalCount = 0;- for (; ; ) {- final int count =- compressor.deflate(compressed, totalCount, compressed.length - totalCount);- totalCount += count;- assert totalCount <= compressed.length;- if (compressor.finished()) {- break;- } else {- compressed = ArrayUtil.grow(compressed);- }- }-- out.writeVInt(totalCount);- out.writeBytes(compressed, totalCount);- }-- @Override- public void compress(byte[] bytes, int off, int len, DataOutput out) throws IOException {- final int dictLength = len / (NUM_SUB_BLOCKS * DICT_SIZE_FACTOR);- final int blockLength = (len - dictLength + NUM_SUB_BLOCKS - 1) / NUM_SUB_BLOCKS;- out.writeVInt(dictLength);- out.writeVInt(blockLength);- final int end = off + len;-- // Compress the dictionary first- compressor.reset();- doCompress(bytes, off, dictLength, out);-- // And then sub blocks- for (int start = off + dictLength; start < end; start += blockLength) {- compressor.reset();- compressor.setDictionary(bytes, off, dictLength);- doCompress(bytes, start, Math.min(blockLength, off + len - start), out);- }- }-- @Override- public void close() throws IOException {- if (closed == false) {- compressor.end();- closed = true;- }- }- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/LZ4WithPresetDictCompressionMode.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/LZ4WithPresetDictCompressionMode.javadeleted file mode 100644index e8d40308d0c..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/LZ4WithPresetDictCompressionMode.java+++ /dev/null@@ -1,201 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene87;--import java.io.IOException;-import org.apache.lucene.backward_codecs.compressing.CompressionMode;-import org.apache.lucene.backward_codecs.compressing.Compressor;-import org.apache.lucene.backward_codecs.compressing.Decompressor;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.index.CorruptIndexException;-import org.apache.lucene.store.ByteBuffersDataOutput;-import org.apache.lucene.store.DataInput;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.compress.LZ4;--/**- * A compression mode that compromises on the compression ratio to provide fast compression and- * decompression.- *- * @lucene.internal- */-public final class LZ4WithPresetDictCompressionMode extends CompressionMode {-- // Shoot for 10 sub blocks- private static final int NUM_SUB_BLOCKS = 10;- // And a dictionary whose size is about 2x smaller than sub blocks- private static final int DICT_SIZE_FACTOR = 2;-- /** Sole constructor. */- public LZ4WithPresetDictCompressionMode() {}-- @Override- public Compressor newCompressor() {- return new LZ4WithPresetDictCompressor();- }-- @Override- public Decompressor newDecompressor() {- return new LZ4WithPresetDictDecompressor();- }-- @Override- public String toString() {- return "BEST_SPEED";- }-- private static final class LZ4WithPresetDictDecompressor extends Decompressor {-- private int[] compressedLengths;- private byte[] buffer;-- LZ4WithPresetDictDecompressor() {- compressedLengths = new int[0];- buffer = new byte[0];- }-- private int readCompressedLengths(- DataInput in, int originalLength, int dictLength, int blockLength) throws IOException {- in.readVInt(); // compressed length of the dictionary, unused- int totalLength = dictLength;- int i = 0;- while (totalLength < originalLength) {- compressedLengths = ArrayUtil.grow(compressedLengths, i + 1);- compressedLengths[i++] = in.readVInt();- totalLength += blockLength;- }- return i;- }-- @Override- public void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes)- throws IOException {- assert offset + length <= originalLength;-- if (length == 0) {- bytes.length = 0;- return;- }-- final int dictLength = in.readVInt();- final int blockLength = in.readVInt();-- final int numBlocks = readCompressedLengths(in, originalLength, dictLength, blockLength);-- buffer = ArrayUtil.grow(buffer, dictLength + blockLength);- bytes.length = 0;- // Read the dictionary- if (LZ4.decompress(EndiannessReverserUtil.wrapDataInput(in), dictLength, buffer, 0)- != dictLength) {- throw new CorruptIndexException("Illegal dict length", in);- }-- int offsetInBlock = dictLength;- int offsetInBytesRef = offset;- if (offset >= dictLength) {- offsetInBytesRef -= dictLength;-- // Skip unneeded blocks- int numBytesToSkip = 0;- for (int i = 0; i < numBlocks && offsetInBlock + blockLength < offset; ++i) {- int compressedBlockLength = compressedLengths[i];- numBytesToSkip += compressedBlockLength;- offsetInBlock += blockLength;- offsetInBytesRef -= blockLength;- }- in.skipBytes(numBytesToSkip);- } else {- // The dictionary contains some bytes we need, copy its content to the BytesRef- bytes.bytes = ArrayUtil.grow(bytes.bytes, dictLength);- System.arraycopy(buffer, 0, bytes.bytes, 0, dictLength);- bytes.length = dictLength;- }-- // Read blocks that intersect with the interval we need- while (offsetInBlock < offset + length) {- final int bytesToDecompress = Math.min(blockLength, offset + length - offsetInBlock);- LZ4.decompress(- EndiannessReverserUtil.wrapDataInput(in), bytesToDecompress, buffer, dictLength);- bytes.bytes = ArrayUtil.grow(bytes.bytes, bytes.length + bytesToDecompress);- System.arraycopy(buffer, dictLength, bytes.bytes, bytes.length, bytesToDecompress);- bytes.length += bytesToDecompress;- offsetInBlock += blockLength;- }-- bytes.offset = offsetInBytesRef;- bytes.length = length;- assert bytes.isValid();- }-- @Override- public Decompressor clone() {- return new LZ4WithPresetDictDecompressor();- }- }-- private static class LZ4WithPresetDictCompressor extends Compressor {-- final ByteBuffersDataOutput compressed;- final LZ4.FastCompressionHashTable hashTable;- byte[] buffer;-- LZ4WithPresetDictCompressor() {- compressed = ByteBuffersDataOutput.newResettableInstance();- hashTable = new LZ4.FastCompressionHashTable();- buffer = BytesRef.EMPTY_BYTES;- }-- private void doCompress(byte[] bytes, int dictLen, int len, DataOutput out) throws IOException {- long prevCompressedSize = compressed.size();- LZ4.compressWithDictionary(bytes, 0, dictLen, len, compressed, hashTable);- // Write the number of compressed bytes- out.writeVInt(Math.toIntExact(compressed.size() - prevCompressedSize));- }-- @Override- public void compress(byte[] bytes, int off, int len, DataOutput out) throws IOException {- final int dictLength = len / (NUM_SUB_BLOCKS * DICT_SIZE_FACTOR);- final int blockLength = (len - dictLength + NUM_SUB_BLOCKS - 1) / NUM_SUB_BLOCKS;- buffer = ArrayUtil.grow(buffer, dictLength + blockLength);- out.writeVInt(dictLength);- out.writeVInt(blockLength);- final int end = off + len;-- compressed.reset();- // Compress the dictionary first- System.arraycopy(bytes, off, buffer, 0, dictLength);- doCompress(buffer, 0, dictLength, out);-- // And then sub blocks- for (int start = off + dictLength; start < end; start += blockLength) {- int l = Math.min(blockLength, off + len - start);- System.arraycopy(bytes, start, buffer, dictLength, l);- doCompress(buffer, dictLength, l, out);- }-- // We only wrote lengths so far, now write compressed data- compressed.copyTo(out);- }-- @Override- public void close() throws IOException {- // no-op- }- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/Lucene87Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/Lucene87Codec.javadeleted file mode 100644index 53a235ab082..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/Lucene87Codec.java+++ /dev/null@@ -1,203 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--package org.apache.lucene.backward_codecs.lucene87;--import java.util.Objects;-import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;-import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;-import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat;-import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;-import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat;-import org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat;-import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat;-import org.apache.lucene.backward_codecs.lucene86.Lucene86PointsFormat;-import org.apache.lucene.backward_codecs.lucene86.Lucene86SegmentInfoFormat;-import org.apache.lucene.codecs.Codec;-import org.apache.lucene.codecs.CompoundFormat;-import org.apache.lucene.codecs.DocValuesFormat;-import org.apache.lucene.codecs.FieldInfosFormat;-import org.apache.lucene.codecs.FilterCodec;-import org.apache.lucene.codecs.KnnVectorsFormat;-import org.apache.lucene.codecs.LiveDocsFormat;-import org.apache.lucene.codecs.NormsFormat;-import org.apache.lucene.codecs.PointsFormat;-import org.apache.lucene.codecs.PostingsFormat;-import org.apache.lucene.codecs.SegmentInfoFormat;-import org.apache.lucene.codecs.StoredFieldsFormat;-import org.apache.lucene.codecs.TermVectorsFormat;-import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;-import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;--/**- * Implements the Lucene 8.6 index format, with configurable per-field postings and docvalues- * formats.- *- * <p>If you want to reuse functionality of this codec in another codec, extend {@link FilterCodec}.- *- * @lucene.experimental- */-public class Lucene87Codec extends Codec {-- /** Configuration option for the codec. */- public enum Mode {- /** Trade compression ratio for retrieval speed. */- BEST_SPEED(Lucene87StoredFieldsFormat.Mode.BEST_SPEED, Lucene80DocValuesFormat.Mode.BEST_SPEED),- /** Trade retrieval speed for compression ratio. */- BEST_COMPRESSION(- Lucene87StoredFieldsFormat.Mode.BEST_COMPRESSION,- Lucene80DocValuesFormat.Mode.BEST_COMPRESSION);-- /** compression mode for stored fields */- protected final Lucene87StoredFieldsFormat.Mode storedMode;-- /** compression mode for doc value fields */- protected final Lucene80DocValuesFormat.Mode dvMode;-- private Mode(Lucene87StoredFieldsFormat.Mode storedMode, Lucene80DocValuesFormat.Mode dvMode) {- this.storedMode = Objects.requireNonNull(storedMode);- this.dvMode = Objects.requireNonNull(dvMode);- }- }-- private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat();- private final FieldInfosFormat fieldInfosFormat = new Lucene60FieldInfosFormat();- private final SegmentInfoFormat segmentInfosFormat = new Lucene86SegmentInfoFormat();- private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();- private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();- private final PointsFormat pointsFormat = new Lucene86PointsFormat();- private final PostingsFormat defaultFormat;-- private final PostingsFormat postingsFormat =- new PerFieldPostingsFormat() {- @Override- public PostingsFormat getPostingsFormatForField(String field) {- return Lucene87Codec.this.getPostingsFormatForField(field);- }- };-- private final DocValuesFormat docValuesFormat =- new PerFieldDocValuesFormat() {- @Override- public DocValuesFormat getDocValuesFormatForField(String field) {- return Lucene87Codec.this.getDocValuesFormatForField(field);- }- };-- private final StoredFieldsFormat storedFieldsFormat;-- /** Instantiates a new codec. */- public Lucene87Codec() {- this(Mode.BEST_COMPRESSION);- }-- /**- * Instantiates a new codec, specifying the compression mode to use.- *- * @param mode compression mode to use for newly flushed/merged segments.- */- public Lucene87Codec(Mode mode) {- super("Lucene87");- this.storedFieldsFormat = new Lucene87StoredFieldsFormat(mode.storedMode);- this.defaultFormat = new Lucene84PostingsFormat();- this.defaultDVFormat = new Lucene80DocValuesFormat(mode.dvMode);- }-- @Override- public StoredFieldsFormat storedFieldsFormat() {- return storedFieldsFormat;- }-- @Override- public TermVectorsFormat termVectorsFormat() {- return vectorsFormat;- }-- @Override- public PostingsFormat postingsFormat() {- return postingsFormat;- }-- @Override- public final FieldInfosFormat fieldInfosFormat() {- return fieldInfosFormat;- }-- @Override- public SegmentInfoFormat segmentInfoFormat() {- return segmentInfosFormat;- }-- @Override- public final LiveDocsFormat liveDocsFormat() {- return liveDocsFormat;- }-- @Override- public CompoundFormat compoundFormat() {- return compoundFormat;- }-- @Override- public PointsFormat pointsFormat() {- return pointsFormat;- }-- @Override- public final KnnVectorsFormat knnVectorsFormat() {- return KnnVectorsFormat.EMPTY;- }-- /**- * Returns the postings format that should be used for writing new segments of <code>field</code>.- *- * <p>The default implementation always returns "Lucene84".- *- * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:- * future version of Lucene are only guaranteed to be able to read the default implementation.- */- public PostingsFormat getPostingsFormatForField(String field) {- return defaultFormat;- }-- /**- * Returns the docvalues format that should be used for writing new segments of <code>field</code>- * .- *- * <p>The default implementation always returns "Lucene80".- *- * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:- * future version of Lucene are only guaranteed to be able to read the default implementation.- */- public DocValuesFormat getDocValuesFormatForField(String field) {- return defaultDVFormat;- }-- @Override- public final DocValuesFormat docValuesFormat() {- return docValuesFormat;- }-- private final DocValuesFormat defaultDVFormat;-- private final NormsFormat normsFormat = new Lucene80NormsFormat();-- @Override- public NormsFormat normsFormat() {- return normsFormat;- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/Lucene87StoredFieldsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/Lucene87StoredFieldsFormat.javadeleted file mode 100644index 988b979354f..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/Lucene87StoredFieldsFormat.java+++ /dev/null@@ -1,176 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene87;--import java.io.IOException;-import java.util.Objects;-import org.apache.lucene.backward_codecs.compressing.CompressionMode;-import org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsFormat;-import org.apache.lucene.codecs.StoredFieldsFormat;-import org.apache.lucene.codecs.StoredFieldsReader;-import org.apache.lucene.codecs.StoredFieldsWriter;-import org.apache.lucene.index.FieldInfos;-import org.apache.lucene.index.SegmentInfo;-import org.apache.lucene.index.StoredFieldVisitor;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.util.packed.DirectMonotonicWriter;--/**- * Lucene 8.7 stored fields format.- *- * <p><b>Principle</b>- *- * <p>This {@link StoredFieldsFormat} compresses blocks of documents in order to improve the- * compression ratio compared to document-level compression. It uses the <a- * href="http://code.google.com/p/lz4/">LZ4</a> compression algorithm by default in 16KB blocks,- * which is fast to compress and very fast to decompress data. Although the default compression- * method that is used ({@link Mode#BEST_SPEED BEST_SPEED}) focuses more on speed than on- * compression ratio, it should provide interesting compression ratios for redundant inputs (such as- * log files, HTML or plain text). For higher compression, you can choose ({@link- * Mode#BEST_COMPRESSION BEST_COMPRESSION}), which uses the <a- * href="http://en.wikipedia.org/wiki/DEFLATE">DEFLATE</a> algorithm with 48kB blocks and shared- * dictionaries for a better ratio at the expense of slower performance. These two options can be- * configured like this:- *- * <pre><code class="language-java">- * // the default: for high performance- * indexWriterConfig.setCodec(new Lucene87Codec(Mode.BEST_SPEED));- * // instead for higher performance (but slower):- * // indexWriterConfig.setCodec(new Lucene87Codec(Mode.BEST_COMPRESSION));- * </code></pre>- *- * <p><b>File formats</b>- *- * <p>Stored fields are represented by three files:- *- * <ol>- * <li><a id="field_data"></a>- * <p>A fields data file (extension <code>.fdt</code>). This file stores a compact- * representation of documents in compressed blocks of 16KB or more. When writing a segment,- * documents are appended to an in-memory <code>byte[]</code> buffer. When its size reaches- * 16KB or more, some metadata about the documents is flushed to disk, immediately followed by- * a compressed representation of the buffer using the <a- * href="https://github.com/lz4/lz4">LZ4</a> <a- * href="http://fastcompression.blogspot.fr/2011/05/lz4-explained.html">compression- * format</a>.- * <p>Notes- * <ul>- * <li>When at least one document in a chunk is large enough so that the chunk is larger- * than 32KB, the chunk will actually be compressed in several LZ4 blocks of 16KB. This- * allows {@link StoredFieldVisitor}s which are only interested in the first fields of a- * document to not have to decompress 10MB of data if the document is 10MB, but only- * 16KB.- * <li>Given that the original lengths are written in the metadata of the chunk, the- * decompressor can leverage this information to stop decoding as soon as enough data- * has been decompressed.- * <li>In case documents are incompressible, the overhead of the compression format is less- * than 0.5%.- * </ul>- * <li><a id="field_index"></a>- * <p>A fields index file (extension <code>.fdx</code>). This file stores two {@link- * DirectMonotonicWriter monotonic arrays}, one for the first doc IDs of each block of- * compressed documents, and another one for the corresponding offsets on disk. At search- * time, the array containing doc IDs is binary-searched in order to find the block that- * contains the expected doc ID, and the associated offset on disk is retrieved from the- * second array.- * <li><a id="field_meta"></a>- * <p>A fields meta file (extension <code>.fdm</code>). This file stores metadata about the- * monotonic arrays stored in the index file.- * </ol>- *- * <p><b>Known limitations</b>- *- * <p>This {@link StoredFieldsFormat} does not support individual documents larger than (<code>- * 2<sup>31</sup> - 2<sup>14</sup></code>) bytes.- *- * @lucene.experimental- */-public class Lucene87StoredFieldsFormat extends StoredFieldsFormat {-- /** Configuration option for stored fields. */- public enum Mode {- /** Trade compression ratio for retrieval speed. */- BEST_SPEED,- /** Trade retrieval speed for compression ratio. */- BEST_COMPRESSION- }-- /** Attribute key for compression mode. */- public static final String MODE_KEY = Lucene87StoredFieldsFormat.class.getSimpleName() + ".mode";-- final Mode mode;-- /** Stored fields format with default options */- public Lucene87StoredFieldsFormat() {- this(Mode.BEST_SPEED);- }-- /** Stored fields format with specified mode */- public Lucene87StoredFieldsFormat(Mode mode) {- this.mode = Objects.requireNonNull(mode);- }-- @Override- public StoredFieldsReader fieldsReader(- Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {- String value = si.getAttribute(MODE_KEY);- if (value == null) {- throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name);- }- Mode mode = Mode.valueOf(value);- return impl(mode).fieldsReader(directory, si, fn, context);- }-- @Override- public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context)- throws IOException {- throw new UnsupportedOperationException("Old codecs may only be used for reading");- }-- StoredFieldsFormat impl(Mode mode) {- switch (mode) {- case BEST_SPEED:- return new Lucene50CompressingStoredFieldsFormat(- "Lucene87StoredFieldsFastData", BEST_SPEED_MODE, BEST_SPEED_BLOCK_LENGTH, 1024, 10);- case BEST_COMPRESSION:- return new Lucene50CompressingStoredFieldsFormat(- "Lucene87StoredFieldsHighData",- BEST_COMPRESSION_MODE,- BEST_COMPRESSION_BLOCK_LENGTH,- 4096,- 10);- default:- throw new AssertionError();- }- }-- // Shoot for 10 sub blocks of 48kB each.- /** Block length for {@link Mode#BEST_COMPRESSION} */- protected static final int BEST_COMPRESSION_BLOCK_LENGTH = 10 * 48 * 1024;-- /** Compression mode for {@link Mode#BEST_COMPRESSION} */- public static final CompressionMode BEST_COMPRESSION_MODE =- new DeflateWithPresetDictCompressionMode();-- // Shoot for 10 sub blocks of 8kB each.- /** Block length for {@link Mode#BEST_SPEED} */- protected static final int BEST_SPEED_BLOCK_LENGTH = 10 * 8 * 1024;-- /** Compression mode for {@link Mode#BEST_SPEED} */- public static final CompressionMode BEST_SPEED_MODE = new LZ4WithPresetDictCompressionMode();-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/package-info.javadeleted file mode 100644index a77730b1b9e..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/package-info.java+++ /dev/null@@ -1,19 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--/** Lucene 8.7 file format. */-package org.apache.lucene.backward_codecs.lucene87;diff --git a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codecindex 15bb6e9fa56..9b7ca1e5149 100644--- a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec+++ b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec@@ -13,10 +13,6 @@# See the License for the specific language governing permissions and# limitations under the License.-org.apache.lucene.backward_codecs.lucene80.Lucene80Codec-org.apache.lucene.backward_codecs.lucene84.Lucene84Codec-org.apache.lucene.backward_codecs.lucene86.Lucene86Codec-org.apache.lucene.backward_codecs.lucene87.Lucene87Codecorg.apache.lucene.backward_codecs.lucene90.Lucene90Codecorg.apache.lucene.backward_codecs.lucene91.Lucene91Codecorg.apache.lucene.backward_codecs.lucene92.Lucene92Codecdiff --git a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormatindex a0c1266b755..87c5b6739a4 100644--- a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat+++ b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat@@ -13,8 +13,6 @@# See the License for the specific language governing permissions and# limitations under the License.-org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat-org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormatorg.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormatorg.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormatorg.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormatdiff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene40/blocktree/Lucene40BlockTreeTermsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene40/blocktree/Lucene40BlockTreeTermsWriter.javadeleted file mode 100644index 6bcc0671a5c..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene40/blocktree/Lucene40BlockTreeTermsWriter.java+++ /dev/null@@ -1,1164 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene40.blocktree;--import java.io.IOException;-import java.util.ArrayList;-import java.util.Arrays;-import java.util.List;-import java.util.Objects;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.BlockTermState;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.FieldsConsumer;-import org.apache.lucene.codecs.NormsProducer;-import org.apache.lucene.codecs.PostingsWriterBase;-import org.apache.lucene.index.FieldInfo;-import org.apache.lucene.index.FieldInfos;-import org.apache.lucene.index.Fields;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.IndexOptions;-import org.apache.lucene.index.SegmentWriteState;-import org.apache.lucene.index.Terms;-import org.apache.lucene.index.TermsEnum;-import org.apache.lucene.store.ByteArrayDataOutput;-import org.apache.lucene.store.ByteBuffersDataOutput;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.store.IndexOutput;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.BytesRefBuilder;-import org.apache.lucene.util.FixedBitSet;-import org.apache.lucene.util.IOUtils;-import org.apache.lucene.util.IntsRefBuilder;-import org.apache.lucene.util.StringHelper;-import org.apache.lucene.util.ToStringUtils;-import org.apache.lucene.util.compress.LZ4;-import org.apache.lucene.util.compress.LowercaseAsciiCompression;-import org.apache.lucene.util.fst.ByteSequenceOutputs;-import org.apache.lucene.util.fst.BytesRefFSTEnum;-import org.apache.lucene.util.fst.FST;-import org.apache.lucene.util.fst.FSTCompiler;-import org.apache.lucene.util.fst.Util;--/*- TODO:-- - Currently there is a one-to-one mapping of indexed- term to term block, but we could decouple the two, ie,- put more terms into the index than there are blocks.- The index would take up more RAM but then it'd be able- to avoid seeking more often and could make PK/FuzzyQ- faster if the additional indexed terms could store- the offset into the terms block.-- - The blocks are not written in true depth-first- order, meaning if you just next() the file pointer will- sometimes jump backwards. For example, block foo* will- be written before block f* because it finished before.- This could possibly hurt performance if the terms dict is- not hot, since OSs anticipate sequential file access. We- could fix the writer to re-order the blocks as a 2nd- pass.-- - Each block encodes the term suffixes packed- sequentially using a separate vInt per term, which is- 1) wasteful and 2) slow (must linear scan to find a- particular suffix). We should instead 1) make- random-access array so we can directly access the Nth- suffix, and 2) bulk-encode this array using bulk int[]- codecs; then at search time we can binary search when- we seek a particular term.-*/--/**- * Block-based terms index and dictionary writer.- *- * <p>Writes terms dict and index, block-encoding (column stride) each term's metadata for each set- * of terms between two index terms.- *- * <p>Files:- *- * <ul>- * <li><code>.tim</code>: <a href="#Termdictionary">Term Dictionary</a>- * <li><code>.tip</code>: <a href="#Termindex">Term Index</a>- * </ul>- *- * <p><a id="Termdictionary"></a>- *- * <h2>Term Dictionary</h2>- *- * <p>The .tim file contains the list of terms in each field along with per-term statistics (such as- * docfreq) and per-term metadata (typically pointers to the postings list for that term in the- * inverted index).- *- * <p>The .tim is arranged in blocks: with blocks containing a variable number of entries (by- * default 25-48), where each entry is either a term or a reference to a sub-block.- *- * <p>NOTE: The term dictionary can plug into different postings implementations: the postings- * writer/reader are actually responsible for encoding and decoding the Postings Metadata and Term- * Metadata sections.- *- * <ul>- * <li>TermsDict (.tim) --> Header, <i>PostingsHeader</i>, NodeBlock<sup>NumBlocks</sup>,- * FieldSummary, DirOffset, Footer- * <li>NodeBlock --> (OuterNode | InnerNode)- * <li>OuterNode --> EntryCount, SuffixLength, Byte<sup>SuffixLength</sup>, StatsLength, <- * TermStats ><sup>EntryCount</sup>, MetaLength,- * <<i>TermMetadata</i>><sup>EntryCount</sup>- * <li>InnerNode --> EntryCount, SuffixLength[,Sub?], Byte<sup>SuffixLength</sup>, StatsLength,- * < TermStats ? ><sup>EntryCount</sup>, MetaLength, <<i>TermMetadata ?- * </i>><sup>EntryCount</sup>- * <li>TermStats --> DocFreq, TotalTermFreq- * <li>FieldSummary --> NumFields, <FieldNumber, NumTerms, RootCodeLength,- * Byte<sup>RootCodeLength</sup>, SumTotalTermFreq?, SumDocFreq, DocCount, LongsSize, MinTerm,- * MaxTerm><sup>NumFields</sup>- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}- * <li>DirOffset --> {@link DataOutput#writeLong Uint64}- * <li>MinTerm,MaxTerm --> {@link DataOutput#writeVInt VInt} length followed by the byte[]- * <li>EntryCount,SuffixLength,StatsLength,DocFreq,MetaLength,NumFields,- * FieldNumber,RootCodeLength,DocCount,LongsSize --> {@link DataOutput#writeVInt VInt}- * <li>TotalTermFreq,NumTerms,SumTotalTermFreq,SumDocFreq --> {@link DataOutput#writeVLong- * VLong}- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}- * </ul>- *- * <p>Notes:- *- * <ul>- * <li>Header is a {@link CodecUtil#writeHeader CodecHeader} storing the version information for- * the BlockTree implementation.- * <li>DirOffset is a pointer to the FieldSummary section.- * <li>DocFreq is the count of documents which contain the term.- * <li>TotalTermFreq is the total number of occurrences of the term. This is encoded as the- * difference between the total number of occurrences and the DocFreq.- * <li>FieldNumber is the fields number from {@link FieldInfos}. (.fnm)- * <li>NumTerms is the number of unique terms for the field.- * <li>RootCode points to the root block for the field.- * <li>SumDocFreq is the total number of postings, the number of term-document pairs across the- * entire field.- * <li>DocCount is the number of documents that have at least one posting for this field.- * <li>LongsSize records how many long values the postings writer/reader record per term (e.g., to- * hold freq/prox/doc file offsets).- * <li>MinTerm, MaxTerm are the lowest and highest term in this field.- * <li>PostingsHeader and TermMetadata are plugged into by the specific postings implementation:- * these contain arbitrary per-file data (such as parameters or versioning information) and- * per-term data (such as pointers to inverted files).- * <li>For inner nodes of the tree, every entry will steal one bit to mark whether it points to- * child nodes(sub-block). If so, the corresponding TermStats and TermMetaData are omitted- * </ul>- *- * <a id="Termindex"></a>- *- * <h2>Term Index</h2>- *- * <p>The .tip file contains an index into the term dictionary, so that it can be accessed randomly.- * The index is also used to determine when a given term cannot exist on disk (in the .tim file),- * saving a disk seek.- *- * <ul>- * <li>TermsIndex (.tip) --> Header, FSTIndex<sup>NumFields</sup>- * <IndexStartFP><sup>NumFields</sup>, DirOffset, Footer- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}- * <li>DirOffset --> {@link DataOutput#writeLong Uint64}- * <li>IndexStartFP --> {@link DataOutput#writeVLong VLong}- * <!-- TODO: better describe FST output here -->- * <li>FSTIndex --> {@link FST FST<byte[]>}- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}- * </ul>- *- * <p>Notes:- *- * <ul>- * <li>The .tip file contains a separate FST for each field. The FST maps a term prefix to the- * on-disk block that holds all terms starting with that prefix. Each field's IndexStartFP- * points to its FST.- * <li>DirOffset is a pointer to the start of the IndexStartFPs for all fields- * <li>It's possible that an on-disk block would contain too many terms (more than the allowed- * maximum (default: 48)). When this happens, the block is sub-divided into new blocks (called- * "floor blocks"), and then the output in the FST for the block's prefix encodes the leading- * byte of each sub-block, and its file pointer.- * </ul>- *- * @see Lucene40BlockTreeTermsReader- * @lucene.experimental- */-public final class Lucene40BlockTreeTermsWriter extends FieldsConsumer {-- /**- * Suggested default value for the {@code minItemsInBlock} parameter to {@link- * #Lucene40BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)}.- */- public static final int DEFAULT_MIN_BLOCK_SIZE = 25;-- /**- * Suggested default value for the {@code maxItemsInBlock} parameter to {@link- * #Lucene40BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)}.- */- public static final int DEFAULT_MAX_BLOCK_SIZE = 48;-- // public static boolean DEBUG = false;- // public static boolean DEBUG2 = false;-- // private final static boolean SAVE_DOT_FILES = false;-- private final IndexOutput metaOut;- private final IndexOutput termsOut;- private final IndexOutput indexOut;- final int maxDoc;- final int minItemsInBlock;- final int maxItemsInBlock;-- final PostingsWriterBase postingsWriter;- final FieldInfos fieldInfos;-- private final List<ByteBuffersDataOutput> fields = new ArrayList<>();-- /**- * Create a new writer. The number of items (terms or sub-blocks) per block will aim to be between- * minItemsPerBlock and maxItemsPerBlock, though in some cases the blocks may be smaller than the- * min.- */- public Lucene40BlockTreeTermsWriter(- SegmentWriteState state,- PostingsWriterBase postingsWriter,- int minItemsInBlock,- int maxItemsInBlock)- throws IOException {- validateSettings(minItemsInBlock, maxItemsInBlock);-- this.minItemsInBlock = minItemsInBlock;- this.maxItemsInBlock = maxItemsInBlock;-- this.maxDoc = state.segmentInfo.maxDoc();- this.fieldInfos = state.fieldInfos;- this.postingsWriter = postingsWriter;-- final String termsName =- IndexFileNames.segmentFileName(- state.segmentInfo.name,- state.segmentSuffix,- Lucene40BlockTreeTermsReader.TERMS_EXTENSION);- termsOut = EndiannessReverserUtil.createOutput(state.directory, termsName, state.context);- boolean success = false;- IndexOutput metaOut = null, indexOut = null;- try {- CodecUtil.writeIndexHeader(- termsOut,- Lucene40BlockTreeTermsReader.TERMS_CODEC_NAME,- Lucene40BlockTreeTermsReader.VERSION_CURRENT,- state.segmentInfo.getId(),- state.segmentSuffix);-- final String indexName =- IndexFileNames.segmentFileName(- state.segmentInfo.name,- state.segmentSuffix,- Lucene40BlockTreeTermsReader.TERMS_INDEX_EXTENSION);- indexOut = EndiannessReverserUtil.createOutput(state.directory, indexName, state.context);- CodecUtil.writeIndexHeader(- indexOut,- Lucene40BlockTreeTermsReader.TERMS_INDEX_CODEC_NAME,- Lucene40BlockTreeTermsReader.VERSION_CURRENT,- state.segmentInfo.getId(),- state.segmentSuffix);- // segment = state.segmentInfo.name;-- final String metaName =- IndexFileNames.segmentFileName(- state.segmentInfo.name,- state.segmentSuffix,- Lucene40BlockTreeTermsReader.TERMS_META_EXTENSION);- metaOut = EndiannessReverserUtil.createOutput(state.directory, metaName, state.context);- CodecUtil.writeIndexHeader(- metaOut,- Lucene40BlockTreeTermsReader.TERMS_META_CODEC_NAME,- Lucene40BlockTreeTermsReader.VERSION_CURRENT,- state.segmentInfo.getId(),- state.segmentSuffix);-- postingsWriter.init(metaOut, state); // have consumer write its format/header-- this.metaOut = metaOut;- this.indexOut = indexOut;- success = true;- } finally {- if (!success) {- IOUtils.closeWhileHandlingException(metaOut, termsOut, indexOut);- }- }- }-- /** Throws {@code IllegalArgumentException} if any of these settings is invalid. */- public static void validateSettings(int minItemsInBlock, int maxItemsInBlock) {- if (minItemsInBlock <= 1) {- throw new IllegalArgumentException("minItemsInBlock must be >= 2; got " + minItemsInBlock);- }- if (minItemsInBlock > maxItemsInBlock) {- throw new IllegalArgumentException(- "maxItemsInBlock must be >= minItemsInBlock; got maxItemsInBlock="- + maxItemsInBlock- + " minItemsInBlock="- + minItemsInBlock);- }- if (2 * (minItemsInBlock - 1) > maxItemsInBlock) {- throw new IllegalArgumentException(- "maxItemsInBlock must be at least 2*(minItemsInBlock-1); got maxItemsInBlock="- + maxItemsInBlock- + " minItemsInBlock="- + minItemsInBlock);- }- }-- @Override- public void write(Fields fields, NormsProducer norms) throws IOException {- // if (DEBUG) System.out.println("\nBTTW.write seg=" + segment);-- String lastField = null;- for (String field : fields) {- assert lastField == null || lastField.compareTo(field) < 0;- lastField = field;-- // if (DEBUG) System.out.println("\nBTTW.write seg=" + segment + " field=" + field);- Terms terms = fields.terms(field);- if (terms == null) {- continue;- }-- TermsEnum termsEnum = terms.iterator();- TermsWriter termsWriter = new TermsWriter(fieldInfos.fieldInfo(field));- while (true) {- BytesRef term = termsEnum.next();- // if (DEBUG) System.out.println("BTTW: next term " + term);-- if (term == null) {- break;- }-- // if (DEBUG) System.out.println("write field=" + fieldInfo.name + " term=" +- // ToStringUtils.bytesRefToString(term));- termsWriter.write(term, termsEnum, norms);- }-- termsWriter.finish();-- // if (DEBUG) System.out.println("\nBTTW.write done seg=" + segment + " field=" + field);- }- }-- static long encodeOutput(long fp, boolean hasTerms, boolean isFloor) {- assert fp < (1L << 62);- return (fp << 2)- | (hasTerms ? Lucene40BlockTreeTermsReader.OUTPUT_FLAG_HAS_TERMS : 0)- | (isFloor ? Lucene40BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR : 0);- }-- private static class PendingEntry {- public final boolean isTerm;-- protected PendingEntry(boolean isTerm) {- this.isTerm = isTerm;- }- }-- private static final class PendingTerm extends PendingEntry {- public final byte[] termBytes;- // stats + metadata- public final BlockTermState state;-- public PendingTerm(BytesRef term, BlockTermState state) {- super(true);- this.termBytes = new byte[term.length];- System.arraycopy(term.bytes, term.offset, termBytes, 0, term.length);- this.state = state;- }-- @Override- public String toString() {- return "TERM: " + ToStringUtils.bytesRefToString(termBytes);- }- }-- private static final class PendingBlock extends PendingEntry {- public final BytesRef prefix;- public final long fp;- public FST<BytesRef> index;- public List<FST<BytesRef>> subIndices;- public final boolean hasTerms;- public final boolean isFloor;- public final int floorLeadByte;-- public PendingBlock(- BytesRef prefix,- long fp,- boolean hasTerms,- boolean isFloor,- int floorLeadByte,- List<FST<BytesRef>> subIndices) {- super(false);- this.prefix = prefix;- this.fp = fp;- this.hasTerms = hasTerms;- this.isFloor = isFloor;- this.floorLeadByte = floorLeadByte;- this.subIndices = subIndices;- }-- @Override- public String toString() {- return "BLOCK: prefix=" + ToStringUtils.bytesRefToString(prefix);- }-- public void compileIndex(- List<PendingBlock> blocks,- ByteBuffersDataOutput scratchBytes,- IntsRefBuilder scratchIntsRef)- throws IOException {-- assert (isFloor && blocks.size() > 1) || (isFloor == false && blocks.size() == 1)- : "isFloor=" + isFloor + " blocks=" + blocks;- assert this == blocks.get(0);-- assert scratchBytes.size() == 0;-- // TODO: try writing the leading vLong in MSB order- // (opposite of what Lucene does today), for better- // outputs sharing in the FST- scratchBytes.writeVLong(encodeOutput(fp, hasTerms, isFloor));- if (isFloor) {- scratchBytes.writeVInt(blocks.size() - 1);- for (int i = 1; i < blocks.size(); i++) {- PendingBlock sub = blocks.get(i);- assert sub.floorLeadByte != -1;- // if (DEBUG) {- // System.out.println(" write floorLeadByte=" +- // Integer.toHexString(sub.floorLeadByte&0xff));- // }- scratchBytes.writeByte((byte) sub.floorLeadByte);- assert sub.fp > fp;- scratchBytes.writeVLong((sub.fp - fp) << 1 | (sub.hasTerms ? 1 : 0));- }- }-- final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();- final FSTCompiler<BytesRef> fstCompiler =- new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs).build();- // if (DEBUG) {- // System.out.println(" compile index for prefix=" + prefix);- // }- // indexBuilder.DEBUG = false;- final byte[] bytes = scratchBytes.toArrayCopy();- assert bytes.length > 0;- fstCompiler.add(Util.toIntsRef(prefix, scratchIntsRef), new BytesRef(bytes, 0, bytes.length));- scratchBytes.reset();-- // Copy over index for all sub-blocks- for (PendingBlock block : blocks) {- if (block.subIndices != null) {- for (FST<BytesRef> subIndex : block.subIndices) {- append(fstCompiler, subIndex, scratchIntsRef);- }- block.subIndices = null;- }- }-- index = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());-- assert subIndices == null;-- /*- Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"));- Util.toDot(index, w, false, false);- System.out.println("SAVED to out.dot");- w.close();- */- }-- // TODO: maybe we could add bulk-add method to- // Builder? Takes FST and unions it w/ current- // FST.- private void append(- FSTCompiler<BytesRef> fstCompiler, FST<BytesRef> subIndex, IntsRefBuilder scratchIntsRef)- throws IOException {- final BytesRefFSTEnum<BytesRef> subIndexEnum = new BytesRefFSTEnum<>(subIndex);- BytesRefFSTEnum.InputOutput<BytesRef> indexEnt;- while ((indexEnt = subIndexEnum.next()) != null) {- // if (DEBUG) {- // System.out.println(" add sub=" + indexEnt.input + " " + indexEnt.input + " output="- // + indexEnt.output);- // }- fstCompiler.add(Util.toIntsRef(indexEnt.input, scratchIntsRef), indexEnt.output);- }- }- }-- private final ByteBuffersDataOutput scratchBytes = ByteBuffersDataOutput.newResettableInstance();- private final IntsRefBuilder scratchIntsRef = new IntsRefBuilder();-- static final BytesRef EMPTY_BYTES_REF = new BytesRef();-- private static class StatsWriter {-- private final DataOutput out;- private final boolean hasFreqs;- private int singletonCount;-- StatsWriter(DataOutput out, boolean hasFreqs) {- this.out = out;- this.hasFreqs = hasFreqs;- }-- void add(int df, long ttf) throws IOException {- // Singletons (DF==1, TTF==1) are run-length encoded- if (df == 1 && (hasFreqs == false || ttf == 1)) {- singletonCount++;- } else {- finish();- out.writeVInt(df << 1);- if (hasFreqs) {- out.writeVLong(ttf - df);- }- }- }-- void finish() throws IOException {- if (singletonCount > 0) {- out.writeVInt(((singletonCount - 1) << 1) | 1);- singletonCount = 0;- }- }- }-- class TermsWriter {- private final FieldInfo fieldInfo;- private long numTerms;- final FixedBitSet docsSeen;- long sumTotalTermFreq;- long sumDocFreq;-- // Records index into pending where the current prefix at that- // length "started"; for example, if current term starts with 't',- // startsByPrefix[0] is the index into pending for the first- // term/sub-block starting with 't'. We use this to figure out when- // to write a new block:- private final BytesRefBuilder lastTerm = new BytesRefBuilder();- private int[] prefixStarts = new int[8];-- // Pending stack of terms and blocks. As terms arrive (in sorted order)- // we append to this stack, and once the top of the stack has enough- // terms starting with a common prefix, we write a new block with- // those terms and replace those terms in the stack with a new block:- private final List<PendingEntry> pending = new ArrayList<>();-- // Reused in writeBlocks:- private final List<PendingBlock> newBlocks = new ArrayList<>();-- private PendingTerm firstPendingTerm;- private PendingTerm lastPendingTerm;-- /** Writes the top count entries in pending, using prevTerm to compute the prefix. */- void writeBlocks(int prefixLength, int count) throws IOException {-- assert count > 0;-- // if (DEBUG2) {- // BytesRef br = new BytesRef(lastTerm.bytes());- // br.length = prefixLength;- // System.out.println("writeBlocks: seg=" + segment + " prefix=" +- // ToStringUtils.bytesRefToString(br) + " count=" + count);- // }-- // Root block better write all remaining pending entries:- assert prefixLength > 0 || count == pending.size();-- int lastSuffixLeadLabel = -1;-- // True if we saw at least one term in this block (we record if a block- // only points to sub-blocks in the terms index so we can avoid seeking- // to it when we are looking for a term):- boolean hasTerms = false;- boolean hasSubBlocks = false;-- int start = pending.size() - count;- int end = pending.size();- int nextBlockStart = start;- int nextFloorLeadLabel = -1;-- for (int i = start; i < end; i++) {-- PendingEntry ent = pending.get(i);-- int suffixLeadLabel;-- if (ent.isTerm) {- PendingTerm term = (PendingTerm) ent;- if (term.termBytes.length == prefixLength) {- // Suffix is 0, i.e. prefix 'foo' and term is- // 'foo' so the term has empty string suffix- // in this block- assert lastSuffixLeadLabel == -1- : "i=" + i + " lastSuffixLeadLabel=" + lastSuffixLeadLabel;- suffixLeadLabel = -1;- } else {- suffixLeadLabel = term.termBytes[prefixLength] & 0xff;- }- } else {- PendingBlock block = (PendingBlock) ent;- assert block.prefix.length > prefixLength;- suffixLeadLabel = block.prefix.bytes[block.prefix.offset + prefixLength] & 0xff;- }- // if (DEBUG) System.out.println(" i=" + i + " ent=" + ent + " suffixLeadLabel=" +- // suffixLeadLabel);-- if (suffixLeadLabel != lastSuffixLeadLabel) {- int itemsInBlock = i - nextBlockStart;- if (itemsInBlock >= minItemsInBlock && end - nextBlockStart > maxItemsInBlock) {- // The count is too large for one block, so we must break it into "floor" blocks, where- // we record- // the leading label of the suffix of the first term in each floor block, so at search- // time we can- // jump to the right floor block. We just use a naive greedy segmenter here: make a new- // floor- // block as soon as we have at least minItemsInBlock. This is not always best: it often- // produces- // a too-small block as the final block:- boolean isFloor = itemsInBlock < count;- newBlocks.add(- writeBlock(- prefixLength,- isFloor,- nextFloorLeadLabel,- nextBlockStart,- i,- hasTerms,- hasSubBlocks));-- hasTerms = false;- hasSubBlocks = false;- nextFloorLeadLabel = suffixLeadLabel;- nextBlockStart = i;- }-- lastSuffixLeadLabel = suffixLeadLabel;- }-- if (ent.isTerm) {- hasTerms = true;- } else {- hasSubBlocks = true;- }- }-- // Write last block, if any:- if (nextBlockStart < end) {- int itemsInBlock = end - nextBlockStart;- boolean isFloor = itemsInBlock < count;- newBlocks.add(- writeBlock(- prefixLength,- isFloor,- nextFloorLeadLabel,- nextBlockStart,- end,- hasTerms,- hasSubBlocks));- }-- assert newBlocks.isEmpty() == false;-- PendingBlock firstBlock = newBlocks.get(0);-- assert firstBlock.isFloor || newBlocks.size() == 1;-- firstBlock.compileIndex(newBlocks, scratchBytes, scratchIntsRef);-- // Remove slice from the top of the pending stack, that we just wrote:- pending.subList(pending.size() - count, pending.size()).clear();-- // Append new block- pending.add(firstBlock);-- newBlocks.clear();- }-- private boolean allEqual(byte[] b, int startOffset, int endOffset, byte value) {- Objects.checkFromToIndex(startOffset, endOffset, b.length);- for (int i = startOffset; i < endOffset; ++i) {- if (b[i] != value) {- return false;- }- }- return true;- }-- /**- * Writes the specified slice (start is inclusive, end is exclusive) from pending stack as a new- * block. If isFloor is true, there were too many (more than maxItemsInBlock) entries sharing- * the same prefix, and so we broke it into multiple floor blocks where we record the starting- * label of the suffix of each floor block.- */- private PendingBlock writeBlock(- int prefixLength,- boolean isFloor,- int floorLeadLabel,- int start,- int end,- boolean hasTerms,- boolean hasSubBlocks)- throws IOException {-- assert end > start;-- long startFP = termsOut.getFilePointer();-- boolean hasFloorLeadLabel = isFloor && floorLeadLabel != -1;-- final BytesRef prefix = new BytesRef(prefixLength + (hasFloorLeadLabel ? 1 : 0));- System.arraycopy(lastTerm.get().bytes, 0, prefix.bytes, 0, prefixLength);- prefix.length = prefixLength;-- // if (DEBUG2) System.out.println(" writeBlock field=" + fieldInfo.name + " prefix=" +- // ToStringUtils.bytesRefToString(prefix) + " fp=" + startFP + " isFloor=" + isFloor +- // " isLastInFloor=" + (end == pending.size()) + " floorLeadLabel=" + floorLeadLabel +- // " start=" + start + " end=" + end + " hasTerms=" + hasTerms + " hasSubBlocks=" +- // hasSubBlocks);-- // Write block header:- int numEntries = end - start;- int code = numEntries << 1;- if (end == pending.size()) {- // Last block:- code |= 1;- }- termsOut.writeVInt(code);-- /*- if (DEBUG) {- System.out.println(" writeBlock " + (isFloor ? "(floor) " : "") + "seg=" + segment + " pending.size()=" +- pending.size() + " prefixLength=" + prefixLength + " indexPrefix=" + ToStringUtils.bytesRefToString(prefix) +- " entCount=" + (end-start+1) + " startFP=" + startFP + (isFloor ? (" floorLeadLabel=" + Integer.toHexString(floorLeadLabel)) : ""));- }- */-- // 1st pass: pack term suffix bytes into byte[] blob- // TODO: cutover to bulk int codec... simple64?-- // We optimize the leaf block case (block has only terms), writing a more- // compact format in this case:- boolean isLeafBlock = hasSubBlocks == false;-- // System.out.println(" isLeaf=" + isLeafBlock);-- final List<FST<BytesRef>> subIndices;-- boolean absolute = true;-- if (isLeafBlock) {- // Block contains only ordinary terms:- subIndices = null;- StatsWriter statsWriter =- new StatsWriter(this.statsWriter, fieldInfo.getIndexOptions() != IndexOptions.DOCS);- for (int i = start; i < end; i++) {- PendingEntry ent = pending.get(i);- assert ent.isTerm : "i=" + i;-- PendingTerm term = (PendingTerm) ent;-- assert StringHelper.startsWith(term.termBytes, prefix) : term + " prefix=" + prefix;- BlockTermState state = term.state;- final int suffix = term.termBytes.length - prefixLength;- // if (DEBUG2) {- // BytesRef suffixBytes = new BytesRef(suffix);- // System.arraycopy(term.termBytes, prefixLength, suffixBytes.bytes, 0, suffix);- // suffixBytes.length = suffix;- // System.out.println(" write term suffix=" +- // ToStringUtils.bytesRefToString(suffixBytes));- // }-- // For leaf block we write suffix straight- suffixLengthsWriter.writeVInt(suffix);- suffixWriter.append(term.termBytes, prefixLength, suffix);- assert floorLeadLabel == -1 || (term.termBytes[prefixLength] & 0xff) >= floorLeadLabel;-- // Write term stats, to separate byte[] blob:- statsWriter.add(state.docFreq, state.totalTermFreq);-- // Write term meta data- postingsWriter.encodeTerm(metaWriter, fieldInfo, state, absolute);- absolute = false;- }- statsWriter.finish();- } else {- // Block has at least one prefix term or a sub block:- subIndices = new ArrayList<>();- StatsWriter statsWriter =- new StatsWriter(this.statsWriter, fieldInfo.getIndexOptions() != IndexOptions.DOCS);- for (int i = start; i < end; i++) {- PendingEntry ent = pending.get(i);- if (ent.isTerm) {- PendingTerm term = (PendingTerm) ent;-- assert StringHelper.startsWith(term.termBytes, prefix) : term + " prefix=" + prefix;- BlockTermState state = term.state;- final int suffix = term.termBytes.length - prefixLength;- // if (DEBUG2) {- // BytesRef suffixBytes = new BytesRef(suffix);- // System.arraycopy(term.termBytes, prefixLength, suffixBytes.bytes, 0, suffix);- // suffixBytes.length = suffix;- // System.out.println(" write term suffix=" +- // ToStringUtils.bytesRefToString(suffixBytes));- // }-- // For non-leaf block we borrow 1 bit to record- // if entry is term or sub-block, and 1 bit to record if- // it's a prefix term. Terms cannot be larger than ~32 KB- // so we won't run out of bits:-- suffixLengthsWriter.writeVInt(suffix << 1);- suffixWriter.append(term.termBytes, prefixLength, suffix);-- // Write term stats, to separate byte[] blob:- statsWriter.add(state.docFreq, state.totalTermFreq);-- // TODO: now that terms dict "sees" these longs,- // we can explore better column-stride encodings- // to encode all long[0]s for this block at- // once, all long[1]s, etc., e.g. using- // Simple64. Alternatively, we could interleave- // stats + meta ... no reason to have them- // separate anymore:-- // Write term meta data- postingsWriter.encodeTerm(metaWriter, fieldInfo, state, absolute);- absolute = false;- } else {- PendingBlock block = (PendingBlock) ent;- assert StringHelper.startsWith(block.prefix, prefix);- final int suffix = block.prefix.length - prefixLength;- assert StringHelper.startsWith(block.prefix, prefix);-- assert suffix > 0;-- // For non-leaf block we borrow 1 bit to record- // if entry is term or sub-block:f- suffixLengthsWriter.writeVInt((suffix << 1) | 1);- suffixWriter.append(block.prefix.bytes, prefixLength, suffix);-- // if (DEBUG2) {- // BytesRef suffixBytes = new BytesRef(suffix);- // System.arraycopy(block.prefix.bytes, prefixLength, suffixBytes.bytes, 0, suffix);- // suffixBytes.length = suffix;- // System.out.println(" write sub-block suffix=" +- // ToStringUtils.bytesRefToString(suffixBytes) +- // " subFP=" + block.fp + " subCode=" + (startFP-block.fp) + " floor=" + block.isFloor);- // }-- assert floorLeadLabel == -1- || (block.prefix.bytes[prefixLength] & 0xff) >= floorLeadLabel- : "floorLeadLabel="- + floorLeadLabel- + " suffixLead="- + (block.prefix.bytes[prefixLength] & 0xff);- assert block.fp < startFP;-- suffixLengthsWriter.writeVLong(startFP - block.fp);- subIndices.add(block.index);- }- }- statsWriter.finish();-- assert subIndices.size() != 0;- }-- // Write suffixes byte[] blob to terms dict output, either uncompressed, compressed with LZ4- // or with LowercaseAsciiCompression.- CompressionAlgorithm compressionAlg = CompressionAlgorithm.NO_COMPRESSION;- // If there are 2 suffix bytes or less per term, then we don't bother compressing as suffix- // are unlikely what- // makes the terms dictionary large, and it also tends to be frequently the case for dense IDs- // like- // auto-increment IDs, so not compressing in that case helps not hurt ID lookups by too much.- // We also only start compressing when the prefix length is greater than 2 since blocks whose- // prefix length is- // 1 or 2 always all get visited when running a fuzzy query whose max number of edits is 2.- if (suffixWriter.length() > 2L * numEntries && prefixLength > 2) {- // LZ4 inserts references whenever it sees duplicate strings of 4 chars or more, so only try- // it out if the- // average suffix length is greater than 6.- if (suffixWriter.length() > 6L * numEntries) {- LZ4.compress(- suffixWriter.bytes(), 0, suffixWriter.length(), spareWriter, compressionHashTable);- if (spareWriter.size() < suffixWriter.length() - (suffixWriter.length() >>> 2)) {- // LZ4 saved more than 25%, go for it- compressionAlg = CompressionAlgorithm.LZ4;- }- }- if (compressionAlg == CompressionAlgorithm.NO_COMPRESSION) {- spareWriter.reset();- if (spareBytes.length < suffixWriter.length()) {- spareBytes = new byte[ArrayUtil.oversize(suffixWriter.length(), 1)];- }- if (LowercaseAsciiCompression.compress(- suffixWriter.bytes(), suffixWriter.length(), spareBytes, spareWriter)) {- compressionAlg = CompressionAlgorithm.LOWERCASE_ASCII;- }- }- }- long token = ((long) suffixWriter.length()) << 3;- if (isLeafBlock) {- token |= 0x04;- }- token |= compressionAlg.code;- termsOut.writeVLong(token);- if (compressionAlg == CompressionAlgorithm.NO_COMPRESSION) {- termsOut.writeBytes(suffixWriter.bytes(), suffixWriter.length());- } else {- spareWriter.copyTo(termsOut);- }- suffixWriter.setLength(0);- spareWriter.reset();-- // Write suffix lengths- final int numSuffixBytes = Math.toIntExact(suffixLengthsWriter.size());- spareBytes = ArrayUtil.grow(spareBytes, numSuffixBytes);- suffixLengthsWriter.copyTo(new ByteArrayDataOutput(spareBytes));- suffixLengthsWriter.reset();- if (allEqual(spareBytes, 1, numSuffixBytes, spareBytes[0])) {- // Structured fields like IDs often have most values of the same length- termsOut.writeVInt((numSuffixBytes << 1) | 1);- termsOut.writeByte(spareBytes[0]);- } else {- termsOut.writeVInt(numSuffixBytes << 1);- termsOut.writeBytes(spareBytes, numSuffixBytes);- }-- // Stats- final int numStatsBytes = Math.toIntExact(statsWriter.size());- termsOut.writeVInt(numStatsBytes);- statsWriter.copyTo(termsOut);- statsWriter.reset();-- // Write term meta data byte[] blob- termsOut.writeVInt((int) metaWriter.size());- metaWriter.copyTo(termsOut);- metaWriter.reset();-- // if (DEBUG) {- // System.out.println(" fpEnd=" + out.getFilePointer());- // }-- if (hasFloorLeadLabel) {- // We already allocated to length+1 above:- prefix.bytes[prefix.length++] = (byte) floorLeadLabel;- }-- return new PendingBlock(prefix, startFP, hasTerms, isFloor, floorLeadLabel, subIndices);- }-- TermsWriter(FieldInfo fieldInfo) {- this.fieldInfo = fieldInfo;- assert fieldInfo.getIndexOptions() != IndexOptions.NONE;- docsSeen = new FixedBitSet(maxDoc);- postingsWriter.setField(fieldInfo);- }-- /** Writes one term's worth of postings. */- public void write(BytesRef text, TermsEnum termsEnum, NormsProducer norms) throws IOException {- /*- if (DEBUG) {- int[] tmp = new int[lastTerm.length];- System.arraycopy(prefixStarts, 0, tmp, 0, tmp.length);- System.out.println("BTTW: write term=" + ToStringUtils.bytesRefToString(text) + " prefixStarts=" + Arrays.toString(tmp) +- " pending.size()=" + pending.size());- }- */-- BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen, norms);- if (state != null) {-- assert state.docFreq != 0;- assert fieldInfo.getIndexOptions() == IndexOptions.DOCS- || state.totalTermFreq >= state.docFreq- : "postingsWriter=" + postingsWriter;- pushTerm(text);-- PendingTerm term = new PendingTerm(text, state);- pending.add(term);- // if (DEBUG) System.out.println(" add pending term = " + text + " pending.size()=" +- // pending.size());-- sumDocFreq += state.docFreq;- sumTotalTermFreq += state.totalTermFreq;- numTerms++;- if (firstPendingTerm == null) {- firstPendingTerm = term;- }- lastPendingTerm = term;- }- }-- /** Pushes the new term to the top of the stack, and writes new blocks. */- private void pushTerm(BytesRef text) throws IOException {- // Find common prefix between last term and current term:- int prefixLength =- Arrays.mismatch(- lastTerm.bytes(),- 0,- lastTerm.length(),- text.bytes,- text.offset,- text.offset + text.length);- if (prefixLength == -1) { // Only happens for the first term, if it is empty- assert lastTerm.length() == 0;- prefixLength = 0;- }-- // if (DEBUG) System.out.println(" shared=" + pos + " lastTerm.length=" + lastTerm.length);-- // Close the "abandoned" suffix now:- for (int i = lastTerm.length() - 1; i >= prefixLength; i--) {-- // How many items on top of the stack share the current suffix- // we are closing:- int prefixTopSize = pending.size() - prefixStarts[i];- if (prefixTopSize >= minItemsInBlock) {- // if (DEBUG) System.out.println("pushTerm i=" + i + " prefixTopSize=" + prefixTopSize +- // " minItemsInBlock=" + minItemsInBlock);- writeBlocks(i + 1, prefixTopSize);- prefixStarts[i] -= prefixTopSize - 1;- }- }-- if (prefixStarts.length < text.length) {- prefixStarts = ArrayUtil.grow(prefixStarts, text.length);- }-- // Init new tail:- for (int i = prefixLength; i < text.length; i++) {- prefixStarts[i] = pending.size();- }-- lastTerm.copyBytes(text);- }-- // Finishes all terms in this field- public void finish() throws IOException {- if (numTerms > 0) {- // if (DEBUG) System.out.println("BTTW: finish prefixStarts=" +- // Arrays.toString(prefixStarts));-- // Add empty term to force closing of all final blocks:- pushTerm(new BytesRef());-- // TODO: if pending.size() is already 1 with a non-zero prefix length- // we can save writing a "degenerate" root block, but we have to- // fix all the places that assume the root block's prefix is the empty string:- pushTerm(new BytesRef());- writeBlocks(0, pending.size());-- // We better have one final "root" block:- assert pending.size() == 1 && !pending.get(0).isTerm- : "pending.size()=" + pending.size() + " pending=" + pending;- final PendingBlock root = (PendingBlock) pending.get(0);- assert root.prefix.length == 0;- final BytesRef rootCode = root.index.getEmptyOutput();- assert rootCode != null;-- ByteBuffersDataOutput metaOut = new ByteBuffersDataOutput();- fields.add(metaOut);-- metaOut.writeVInt(fieldInfo.number);- metaOut.writeVLong(numTerms);- metaOut.writeVInt(rootCode.length);- metaOut.writeBytes(rootCode.bytes, rootCode.offset, rootCode.length);- assert fieldInfo.getIndexOptions() != IndexOptions.NONE;- if (fieldInfo.getIndexOptions() != IndexOptions.DOCS) {- metaOut.writeVLong(sumTotalTermFreq);- }- metaOut.writeVLong(sumDocFreq);- metaOut.writeVInt(docsSeen.cardinality());- writeBytesRef(metaOut, new BytesRef(firstPendingTerm.termBytes));- writeBytesRef(metaOut, new BytesRef(lastPendingTerm.termBytes));- metaOut.writeVLong(indexOut.getFilePointer());- // Write FST to index- root.index.save(metaOut, indexOut);- // System.out.println(" write FST " + indexStartFP + " field=" + fieldInfo.name);-- /*- if (DEBUG) {- final String dotFileName = segment + "_" + fieldInfo.name + ".dot";- Writer w = new OutputStreamWriter(new FileOutputStream(dotFileName));- Util.toDot(root.index, w, false, false);- System.out.println("SAVED to " + dotFileName);- w.close();- }- */-- } else {- assert sumTotalTermFreq == 0- || fieldInfo.getIndexOptions() == IndexOptions.DOCS && sumTotalTermFreq == -1;- assert sumDocFreq == 0;- assert docsSeen.cardinality() == 0;- }- }-- private final ByteBuffersDataOutput suffixLengthsWriter =- ByteBuffersDataOutput.newResettableInstance();- private final BytesRefBuilder suffixWriter = new BytesRefBuilder();- private final ByteBuffersDataOutput statsWriter = ByteBuffersDataOutput.newResettableInstance();- private final ByteBuffersDataOutput metaWriter = ByteBuffersDataOutput.newResettableInstance();- private final ByteBuffersDataOutput spareWriter = ByteBuffersDataOutput.newResettableInstance();- private byte[] spareBytes = BytesRef.EMPTY_BYTES;- private final LZ4.HighCompressionHashTable compressionHashTable =- new LZ4.HighCompressionHashTable();- }-- private boolean closed;-- @Override- public void close() throws IOException {- if (closed) {- return;- }- closed = true;-- boolean success = false;- try {- metaOut.writeVInt(fields.size());- for (ByteBuffersDataOutput fieldMeta : fields) {- fieldMeta.copyTo(metaOut);- }- CodecUtil.writeFooter(indexOut);- metaOut.writeLong(indexOut.getFilePointer());- CodecUtil.writeFooter(termsOut);- metaOut.writeLong(termsOut.getFilePointer());- CodecUtil.writeFooter(metaOut);- success = true;- } finally {- if (success) {- IOUtils.close(metaOut, termsOut, indexOut, postingsWriter);- } else {- IOUtils.closeWhileHandlingException(metaOut, termsOut, indexOut, postingsWriter);- }- }- }-- private static void writeBytesRef(DataOutput out, BytesRef bytes) throws IOException {- out.writeVInt(bytes.length);- out.writeBytes(bytes.bytes, bytes.offset, bytes.length);- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene40/blocktree/Lucene40BlockTreeTermsWriterV5.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene40/blocktree/Lucene40BlockTreeTermsWriterV5.javadeleted file mode 100644index 165f296f282..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene40/blocktree/Lucene40BlockTreeTermsWriterV5.java+++ /dev/null@@ -1,1096 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene40.blocktree;--import java.io.IOException;-import java.util.ArrayList;-import java.util.Arrays;-import java.util.List;-import java.util.Objects;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.BlockTermState;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.FieldsConsumer;-import org.apache.lucene.codecs.NormsProducer;-import org.apache.lucene.codecs.PostingsWriterBase;-import org.apache.lucene.index.FieldInfo;-import org.apache.lucene.index.FieldInfos;-import org.apache.lucene.index.Fields;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.IndexOptions;-import org.apache.lucene.index.SegmentWriteState;-import org.apache.lucene.index.Terms;-import org.apache.lucene.index.TermsEnum;-import org.apache.lucene.store.ByteArrayDataOutput;-import org.apache.lucene.store.ByteBuffersDataOutput;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.store.IndexOutput;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.BytesRefBuilder;-import org.apache.lucene.util.FixedBitSet;-import org.apache.lucene.util.IOUtils;-import org.apache.lucene.util.IntsRefBuilder;-import org.apache.lucene.util.StringHelper;-import org.apache.lucene.util.compress.LZ4;-import org.apache.lucene.util.compress.LowercaseAsciiCompression;-import org.apache.lucene.util.fst.ByteSequenceOutputs;-import org.apache.lucene.util.fst.BytesRefFSTEnum;-import org.apache.lucene.util.fst.FST;-import org.apache.lucene.util.fst.FSTCompiler;-import org.apache.lucene.util.fst.Util;--/**- * Writer for {@link Lucene40BlockTreeTermsReader} prior to the refactoring that separated metadata- * to its own file.- *- * @see Lucene40BlockTreeTermsReader#VERSION_META_FILE- */-public final class Lucene40BlockTreeTermsWriterV5 extends FieldsConsumer {-- /**- * Suggested default value for the {@code minItemsInBlock} parameter to {@link- * #Lucene40BlockTreeTermsWriterV5(SegmentWriteState,PostingsWriterBase,int,int)}.- */- public static final int DEFAULT_MIN_BLOCK_SIZE = 25;-- /**- * Suggested default value for the {@code maxItemsInBlock} parameter to {@link- * #Lucene40BlockTreeTermsWriterV5(SegmentWriteState,PostingsWriterBase,int,int)}.- */- public static final int DEFAULT_MAX_BLOCK_SIZE = 48;-- private static final int VERSION_CURRENT = Lucene40BlockTreeTermsReader.VERSION_META_FILE - 1;-- // public static boolean DEBUG = false;- // public static boolean DEBUG2 = false;-- // private final static boolean SAVE_DOT_FILES = false;-- private final IndexOutput termsOut;- private final IndexOutput indexOut;- final int maxDoc;- final int minItemsInBlock;- final int maxItemsInBlock;-- final PostingsWriterBase postingsWriter;- final FieldInfos fieldInfos;-- private static class FieldMetaData {- public final FieldInfo fieldInfo;- public final BytesRef rootCode;- public final long numTerms;- public final long indexStartFP;- public final long sumTotalTermFreq;- public final long sumDocFreq;- public final int docCount;- public final BytesRef minTerm;- public final BytesRef maxTerm;-- public FieldMetaData(- FieldInfo fieldInfo,- BytesRef rootCode,- long numTerms,- long indexStartFP,- long sumTotalTermFreq,- long sumDocFreq,- int docCount,- BytesRef minTerm,- BytesRef maxTerm) {- assert numTerms > 0;- this.fieldInfo = fieldInfo;- assert rootCode != null : "field=" + fieldInfo.name + " numTerms=" + numTerms;- this.rootCode = rootCode;- this.indexStartFP = indexStartFP;- this.numTerms = numTerms;- this.sumTotalTermFreq = sumTotalTermFreq;- this.sumDocFreq = sumDocFreq;- this.docCount = docCount;- this.minTerm = minTerm;- this.maxTerm = maxTerm;- }- }-- private final List<FieldMetaData> fields = new ArrayList<>();-- /**- * Create a new writer. The number of items (terms or sub-blocks) per block will aim to be between- * minItemsPerBlock and maxItemsPerBlock, though in some cases the blocks may be smaller than the- * min.- */- public Lucene40BlockTreeTermsWriterV5(- SegmentWriteState state,- PostingsWriterBase postingsWriter,- int minItemsInBlock,- int maxItemsInBlock)- throws IOException {- validateSettings(minItemsInBlock, maxItemsInBlock);-- this.minItemsInBlock = minItemsInBlock;- this.maxItemsInBlock = maxItemsInBlock;-- this.maxDoc = state.segmentInfo.maxDoc();- this.fieldInfos = state.fieldInfos;- this.postingsWriter = postingsWriter;-- final String termsName =- IndexFileNames.segmentFileName(- state.segmentInfo.name,- state.segmentSuffix,- Lucene40BlockTreeTermsReader.TERMS_EXTENSION);- termsOut = EndiannessReverserUtil.createOutput(state.directory, termsName, state.context);- boolean success = false;- IndexOutput indexOut = null;- try {- CodecUtil.writeIndexHeader(- termsOut,- Lucene40BlockTreeTermsReader.TERMS_CODEC_NAME,- VERSION_CURRENT,- state.segmentInfo.getId(),- state.segmentSuffix);-- final String indexName =- IndexFileNames.segmentFileName(- state.segmentInfo.name,- state.segmentSuffix,- Lucene40BlockTreeTermsReader.TERMS_INDEX_EXTENSION);- indexOut = EndiannessReverserUtil.createOutput(state.directory, indexName, state.context);- CodecUtil.writeIndexHeader(- indexOut,- Lucene40BlockTreeTermsReader.TERMS_INDEX_CODEC_NAME,- VERSION_CURRENT,- state.segmentInfo.getId(),- state.segmentSuffix);- // segment = state.segmentInfo.name;-- postingsWriter.init(termsOut, state); // have consumer write its format/header-- this.indexOut = indexOut;- success = true;- } finally {- if (!success) {- IOUtils.closeWhileHandlingException(termsOut, indexOut);- }- }- }-- /** Writes the terms file trailer. */- private void writeTrailer(IndexOutput out, long dirStart) throws IOException {- out.writeLong(dirStart);- }-- /** Writes the index file trailer. */- private void writeIndexTrailer(IndexOutput indexOut, long dirStart) throws IOException {- indexOut.writeLong(dirStart);- }-- /** Throws {@code IllegalArgumentException} if any of these settings is invalid. */- public static void validateSettings(int minItemsInBlock, int maxItemsInBlock) {- if (minItemsInBlock <= 1) {- throw new IllegalArgumentException("minItemsInBlock must be >= 2; got " + minItemsInBlock);- }- if (minItemsInBlock > maxItemsInBlock) {- throw new IllegalArgumentException(- "maxItemsInBlock must be >= minItemsInBlock; got maxItemsInBlock="- + maxItemsInBlock- + " minItemsInBlock="- + minItemsInBlock);- }- if (2 * (minItemsInBlock - 1) > maxItemsInBlock) {- throw new IllegalArgumentException(- "maxItemsInBlock must be at least 2*(minItemsInBlock-1); got maxItemsInBlock="- + maxItemsInBlock- + " minItemsInBlock="- + minItemsInBlock);- }- }-- @Override- public void write(Fields fields, NormsProducer norms) throws IOException {- // if (DEBUG) System.out.println("\nBTTW.write seg=" + segment);-- String lastField = null;- for (String field : fields) {- assert lastField == null || lastField.compareTo(field) < 0;- lastField = field;-- // if (DEBUG) System.out.println("\nBTTW.write seg=" + segment + " field=" + field);- Terms terms = fields.terms(field);- if (terms == null) {- continue;- }-- TermsEnum termsEnum = terms.iterator();- TermsWriter termsWriter = new TermsWriter(fieldInfos.fieldInfo(field));- while (true) {- BytesRef term = termsEnum.next();- // if (DEBUG) System.out.println("BTTW: next term " + term);-- if (term == null) {- break;- }-- // if (DEBUG) System.out.println("write field=" + fieldInfo.name + " term=" +- // brToString(term));- termsWriter.write(term, termsEnum, norms);- }-- termsWriter.finish();-- // if (DEBUG) System.out.println("\nBTTW.write done seg=" + segment + " field=" + field);- }- }-- static long encodeOutput(long fp, boolean hasTerms, boolean isFloor) {- assert fp < (1L << 62);- return (fp << 2)- | (hasTerms ? Lucene40BlockTreeTermsReader.OUTPUT_FLAG_HAS_TERMS : 0)- | (isFloor ? Lucene40BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR : 0);- }-- private static class PendingEntry {- public final boolean isTerm;-- protected PendingEntry(boolean isTerm) {- this.isTerm = isTerm;- }- }-- private static final class PendingTerm extends PendingEntry {- public final byte[] termBytes;- // stats + metadata- public final BlockTermState state;-- public PendingTerm(BytesRef term, BlockTermState state) {- super(true);- this.termBytes = new byte[term.length];- System.arraycopy(term.bytes, term.offset, termBytes, 0, term.length);- this.state = state;- }-- @Override- public String toString() {- return "TERM: " + brToString(termBytes);- }- }-- // for debugging- @SuppressWarnings("unused")- static String brToString(BytesRef b) {- if (b == null) {- return "(null)";- } else {- try {- return b.utf8ToString() + " " + b;- } catch (Throwable t) {- // If BytesRef isn't actually UTF8, or it's eg a- // prefix of UTF8 that ends mid-unicode-char, we- // fallback to hex:- return b.toString();- }- }- }-- // for debugging- @SuppressWarnings("unused")- static String brToString(byte[] b) {- return brToString(new BytesRef(b));- }-- private static final class PendingBlock extends PendingEntry {- public final BytesRef prefix;- public final long fp;- public FST<BytesRef> index;- public List<FST<BytesRef>> subIndices;- public final boolean hasTerms;- public final boolean isFloor;- public final int floorLeadByte;-- public PendingBlock(- BytesRef prefix,- long fp,- boolean hasTerms,- boolean isFloor,- int floorLeadByte,- List<FST<BytesRef>> subIndices) {- super(false);- this.prefix = prefix;- this.fp = fp;- this.hasTerms = hasTerms;- this.isFloor = isFloor;- this.floorLeadByte = floorLeadByte;- this.subIndices = subIndices;- }-- @Override- public String toString() {- return "BLOCK: prefix=" + brToString(prefix);- }-- public void compileIndex(- List<PendingBlock> blocks,- ByteBuffersDataOutput scratchBytes,- IntsRefBuilder scratchIntsRef)- throws IOException {-- assert (isFloor && blocks.size() > 1) || (isFloor == false && blocks.size() == 1)- : "isFloor=" + isFloor + " blocks=" + blocks;- assert this == blocks.get(0);-- assert scratchBytes.size() == 0;-- // TODO: try writing the leading vLong in MSB order- // (opposite of what Lucene does today), for better- // outputs sharing in the FST- scratchBytes.writeVLong(encodeOutput(fp, hasTerms, isFloor));- if (isFloor) {- scratchBytes.writeVInt(blocks.size() - 1);- for (int i = 1; i < blocks.size(); i++) {- PendingBlock sub = blocks.get(i);- assert sub.floorLeadByte != -1;- // if (DEBUG) {- // System.out.println(" write floorLeadByte=" +- // Integer.toHexString(sub.floorLeadByte&0xff));- // }- scratchBytes.writeByte((byte) sub.floorLeadByte);- assert sub.fp > fp;- scratchBytes.writeVLong((sub.fp - fp) << 1 | (sub.hasTerms ? 1 : 0));- }- }-- final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();- final FSTCompiler<BytesRef> fstCompiler =- new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs).build();- // if (DEBUG) {- // System.out.println(" compile index for prefix=" + prefix);- // }- // indexBuilder.DEBUG = false;- final byte[] bytes = scratchBytes.toArrayCopy();- assert bytes.length > 0;- fstCompiler.add(Util.toIntsRef(prefix, scratchIntsRef), new BytesRef(bytes, 0, bytes.length));- scratchBytes.reset();-- // Copy over index for all sub-blocks- for (PendingBlock block : blocks) {- if (block.subIndices != null) {- for (FST<BytesRef> subIndex : block.subIndices) {- append(fstCompiler, subIndex, scratchIntsRef);- }- block.subIndices = null;- }- }-- index = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());-- assert subIndices == null;-- /*- Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"));- Util.toDot(index, w, false, false);- System.out.println("SAVED to out.dot");- w.close();- */- }-- // TODO: maybe we could add bulk-add method to- // Builder? Takes FST and unions it w/ current- // FST.- private void append(- FSTCompiler<BytesRef> fstCompiler, FST<BytesRef> subIndex, IntsRefBuilder scratchIntsRef)- throws IOException {- final BytesRefFSTEnum<BytesRef> subIndexEnum = new BytesRefFSTEnum<>(subIndex);- BytesRefFSTEnum.InputOutput<BytesRef> indexEnt;- while ((indexEnt = subIndexEnum.next()) != null) {- // if (DEBUG) {- // System.out.println(" add sub=" + indexEnt.input + " " + indexEnt.input + " output="- // + indexEnt.output);- // }- fstCompiler.add(Util.toIntsRef(indexEnt.input, scratchIntsRef), indexEnt.output);- }- }- }-- private final ByteBuffersDataOutput scratchBytes = ByteBuffersDataOutput.newResettableInstance();- private final IntsRefBuilder scratchIntsRef = new IntsRefBuilder();-- static final BytesRef EMPTY_BYTES_REF = new BytesRef();-- private static class StatsWriter {-- private final DataOutput out;- private final boolean hasFreqs;- private int singletonCount;-- StatsWriter(DataOutput out, boolean hasFreqs) {- this.out = out;- this.hasFreqs = hasFreqs;- }-- void add(int df, long ttf) throws IOException {- // Singletons (DF==1, TTF==1) are run-length encoded- if (df == 1 && (hasFreqs == false || ttf == 1)) {- singletonCount++;- } else {- finish();- out.writeVInt(df << 1);- if (hasFreqs) {- out.writeVLong(ttf - df);- }- }- }-- void finish() throws IOException {- if (singletonCount > 0) {- out.writeVInt(((singletonCount - 1) << 1) | 1);- singletonCount = 0;- }- }- }-- class TermsWriter {- private final FieldInfo fieldInfo;- private long numTerms;- final FixedBitSet docsSeen;- long sumTotalTermFreq;- long sumDocFreq;- long indexStartFP;-- // Records index into pending where the current prefix at that- // length "started"; for example, if current term starts with 't',- // startsByPrefix[0] is the index into pending for the first- // term/sub-block starting with 't'. We use this to figure out when- // to write a new block:- private final BytesRefBuilder lastTerm = new BytesRefBuilder();- private int[] prefixStarts = new int[8];-- // Pending stack of terms and blocks. As terms arrive (in sorted order)- // we append to this stack, and once the top of the stack has enough- // terms starting with a common prefix, we write a new block with- // those terms and replace those terms in the stack with a new block:- private final List<PendingEntry> pending = new ArrayList<>();-- // Reused in writeBlocks:- private final List<PendingBlock> newBlocks = new ArrayList<>();-- private PendingTerm firstPendingTerm;- private PendingTerm lastPendingTerm;-- /** Writes the top count entries in pending, using prevTerm to compute the prefix. */- void writeBlocks(int prefixLength, int count) throws IOException {-- assert count > 0;-- // if (DEBUG2) {- // BytesRef br = new BytesRef(lastTerm.bytes());- // br.length = prefixLength;- // System.out.println("writeBlocks: seg=" + segment + " prefix=" + brToString(br) + " count="- // + count);- // }-- // Root block better write all remaining pending entries:- assert prefixLength > 0 || count == pending.size();-- int lastSuffixLeadLabel = -1;-- // True if we saw at least one term in this block (we record if a block- // only points to sub-blocks in the terms index so we can avoid seeking- // to it when we are looking for a term):- boolean hasTerms = false;- boolean hasSubBlocks = false;-- int start = pending.size() - count;- int end = pending.size();- int nextBlockStart = start;- int nextFloorLeadLabel = -1;-- for (int i = start; i < end; i++) {-- PendingEntry ent = pending.get(i);-- int suffixLeadLabel;-- if (ent.isTerm) {- PendingTerm term = (PendingTerm) ent;- if (term.termBytes.length == prefixLength) {- // Suffix is 0, i.e. prefix 'foo' and term is- // 'foo' so the term has empty string suffix- // in this block- assert lastSuffixLeadLabel == -1- : "i=" + i + " lastSuffixLeadLabel=" + lastSuffixLeadLabel;- suffixLeadLabel = -1;- } else {- suffixLeadLabel = term.termBytes[prefixLength] & 0xff;- }- } else {- PendingBlock block = (PendingBlock) ent;- assert block.prefix.length > prefixLength;- suffixLeadLabel = block.prefix.bytes[block.prefix.offset + prefixLength] & 0xff;- }- // if (DEBUG) System.out.println(" i=" + i + " ent=" + ent + " suffixLeadLabel=" +- // suffixLeadLabel);-- if (suffixLeadLabel != lastSuffixLeadLabel) {- int itemsInBlock = i - nextBlockStart;- if (itemsInBlock >= minItemsInBlock && end - nextBlockStart > maxItemsInBlock) {- // The count is too large for one block, so we must break it into "floor" blocks, where- // we record- // the leading label of the suffix of the first term in each floor block, so at search- // time we can- // jump to the right floor block. We just use a naive greedy segmenter here: make a new- // floor- // block as soon as we have at least minItemsInBlock. This is not always best: it often- // produces- // a too-small block as the final block:- boolean isFloor = itemsInBlock < count;- newBlocks.add(- writeBlock(- prefixLength,- isFloor,- nextFloorLeadLabel,- nextBlockStart,- i,- hasTerms,- hasSubBlocks));-- hasTerms = false;- hasSubBlocks = false;- nextFloorLeadLabel = suffixLeadLabel;- nextBlockStart = i;- }-- lastSuffixLeadLabel = suffixLeadLabel;- }-- if (ent.isTerm) {- hasTerms = true;- } else {- hasSubBlocks = true;- }- }-- // Write last block, if any:- if (nextBlockStart < end) {- int itemsInBlock = end - nextBlockStart;- boolean isFloor = itemsInBlock < count;- newBlocks.add(- writeBlock(- prefixLength,- isFloor,- nextFloorLeadLabel,- nextBlockStart,- end,- hasTerms,- hasSubBlocks));- }-- assert newBlocks.isEmpty() == false;-- PendingBlock firstBlock = newBlocks.get(0);-- assert firstBlock.isFloor || newBlocks.size() == 1;-- firstBlock.compileIndex(newBlocks, scratchBytes, scratchIntsRef);-- // Remove slice from the top of the pending stack, that we just wrote:- pending.subList(pending.size() - count, pending.size()).clear();-- // Append new block- pending.add(firstBlock);-- newBlocks.clear();- }-- private boolean allEqual(byte[] b, int startOffset, int endOffset, byte value) {- Objects.checkFromToIndex(startOffset, endOffset, b.length);- for (int i = startOffset; i < endOffset; ++i) {- if (b[i] != value) {- return false;- }- }- return true;- }-- /**- * Writes the specified slice (start is inclusive, end is exclusive) from pending stack as a new- * block. If isFloor is true, there were too many (more than maxItemsInBlock) entries sharing- * the same prefix, and so we broke it into multiple floor blocks where we record the starting- * label of the suffix of each floor block.- */- private PendingBlock writeBlock(- int prefixLength,- boolean isFloor,- int floorLeadLabel,- int start,- int end,- boolean hasTerms,- boolean hasSubBlocks)- throws IOException {-- assert end > start;-- long startFP = termsOut.getFilePointer();-- boolean hasFloorLeadLabel = isFloor && floorLeadLabel != -1;-- final BytesRef prefix = new BytesRef(prefixLength + (hasFloorLeadLabel ? 1 : 0));- System.arraycopy(lastTerm.get().bytes, 0, prefix.bytes, 0, prefixLength);- prefix.length = prefixLength;-- // if (DEBUG2) System.out.println(" writeBlock field=" + fieldInfo.name + " prefix=" +- // brToString(prefix) + " fp=" + startFP + " isFloor=" + isFloor + " isLastInFloor=" + (end ==- // pending.size()) + " floorLeadLabel=" + floorLeadLabel + " start=" + start + " end=" + end +- // " hasTerms=" + hasTerms + " hasSubBlocks=" + hasSubBlocks);-- // Write block header:- int numEntries = end - start;- int code = numEntries << 1;- if (end == pending.size()) {- // Last block:- code |= 1;- }- termsOut.writeVInt(code);-- /*- if (DEBUG) {- System.out.println(" writeBlock " + (isFloor ? "(floor) " : "") + "seg=" + segment + " pending.size()=" + pending.size() + " prefixLength=" + prefixLength + " indexPrefix=" + brToString(prefix) + " entCount=" + (end-start+1) + " startFP=" + startFP + (isFloor ? (" floorLeadLabel=" + Integer.toHexString(floorLeadLabel)) : ""));- }- */-- // 1st pass: pack term suffix bytes into byte[] blob- // TODO: cutover to bulk int codec... simple64?-- // We optimize the leaf block case (block has only terms), writing a more- // compact format in this case:- boolean isLeafBlock = hasSubBlocks == false;-- // System.out.println(" isLeaf=" + isLeafBlock);-- final List<FST<BytesRef>> subIndices;-- boolean absolute = true;-- if (isLeafBlock) {- // Block contains only ordinary terms:- subIndices = null;- StatsWriter statsWriter =- new StatsWriter(this.statsWriter, fieldInfo.getIndexOptions() != IndexOptions.DOCS);- for (int i = start; i < end; i++) {- PendingEntry ent = pending.get(i);- assert ent.isTerm : "i=" + i;-- PendingTerm term = (PendingTerm) ent;-- assert StringHelper.startsWith(term.termBytes, prefix)- : "term.term=" + new BytesRef(term.termBytes) + " prefix=" + prefix;- BlockTermState state = term.state;- final int suffix = term.termBytes.length - prefixLength;- // if (DEBUG2) {- // BytesRef suffixBytes = new BytesRef(suffix);- // System.arraycopy(term.termBytes, prefixLength, suffixBytes.bytes, 0, suffix);- // suffixBytes.length = suffix;- // System.out.println(" write term suffix=" + brToString(suffixBytes));- // }-- // For leaf block we write suffix straight- suffixLengthsWriter.writeVInt(suffix);- suffixWriter.append(term.termBytes, prefixLength, suffix);- assert floorLeadLabel == -1 || (term.termBytes[prefixLength] & 0xff) >= floorLeadLabel;-- // Write term stats, to separate byte[] blob:- statsWriter.add(state.docFreq, state.totalTermFreq);-- // Write term meta data- postingsWriter.encodeTerm(metaWriter, fieldInfo, state, absolute);- absolute = false;- }- statsWriter.finish();- } else {- // Block has at least one prefix term or a sub block:- subIndices = new ArrayList<>();- StatsWriter statsWriter =- new StatsWriter(this.statsWriter, fieldInfo.getIndexOptions() != IndexOptions.DOCS);- for (int i = start; i < end; i++) {- PendingEntry ent = pending.get(i);- if (ent.isTerm) {- PendingTerm term = (PendingTerm) ent;-- assert StringHelper.startsWith(term.termBytes, prefix)- : "term.term=" + new BytesRef(term.termBytes) + " prefix=" + prefix;- BlockTermState state = term.state;- final int suffix = term.termBytes.length - prefixLength;- // if (DEBUG2) {- // BytesRef suffixBytes = new BytesRef(suffix);- // System.arraycopy(term.termBytes, prefixLength, suffixBytes.bytes, 0, suffix);- // suffixBytes.length = suffix;- // System.out.println(" write term suffix=" + brToString(suffixBytes));- // }-- // For non-leaf block we borrow 1 bit to record- // if entry is term or sub-block, and 1 bit to record if- // it's a prefix term. Terms cannot be larger than ~32 KB- // so we won't run out of bits:-- suffixLengthsWriter.writeVInt(suffix << 1);- suffixWriter.append(term.termBytes, prefixLength, suffix);-- // Write term stats, to separate byte[] blob:- statsWriter.add(state.docFreq, state.totalTermFreq);-- // TODO: now that terms dict "sees" these longs,- // we can explore better column-stride encodings- // to encode all long[0]s for this block at- // once, all long[1]s, etc., e.g. using- // Simple64. Alternatively, we could interleave- // stats + meta ... no reason to have them- // separate anymore:-- // Write term meta data- postingsWriter.encodeTerm(metaWriter, fieldInfo, state, absolute);- absolute = false;- } else {- PendingBlock block = (PendingBlock) ent;- assert StringHelper.startsWith(block.prefix, prefix);- final int suffix = block.prefix.length - prefixLength;- assert StringHelper.startsWith(block.prefix, prefix);-- assert suffix > 0;-- // For non-leaf block we borrow 1 bit to record- // if entry is term or sub-block:f- suffixLengthsWriter.writeVInt((suffix << 1) | 1);- suffixWriter.append(block.prefix.bytes, prefixLength, suffix);-- // if (DEBUG2) {- // BytesRef suffixBytes = new BytesRef(suffix);- // System.arraycopy(block.prefix.bytes, prefixLength, suffixBytes.bytes, 0, suffix);- // suffixBytes.length = suffix;- // System.out.println(" write sub-block suffix=" + brToString(suffixBytes) + "- // subFP=" + block.fp + " subCode=" + (startFP-block.fp) + " floor=" + block.isFloor);- // }-- assert floorLeadLabel == -1- || (block.prefix.bytes[prefixLength] & 0xff) >= floorLeadLabel- : "floorLeadLabel="- + floorLeadLabel- + " suffixLead="- + (block.prefix.bytes[prefixLength] & 0xff);- assert block.fp < startFP;-- suffixLengthsWriter.writeVLong(startFP - block.fp);- subIndices.add(block.index);- }- }- statsWriter.finish();-- assert subIndices.size() != 0;- }-- // Write suffixes byte[] blob to terms dict output, either uncompressed, compressed with LZ4- // or with LowercaseAsciiCompression.- CompressionAlgorithm compressionAlg = CompressionAlgorithm.NO_COMPRESSION;- // If there are 2 suffix bytes or less per term, then we don't bother compressing as suffix- // are unlikely what- // makes the terms dictionary large, and it also tends to be frequently the case for dense IDs- // like- // auto-increment IDs, so not compressing in that case helps not hurt ID lookups by too much.- // We also only start compressing when the prefix length is greater than 2 since blocks whose- // prefix length is- // 1 or 2 always all get visited when running a fuzzy query whose max number of edits is 2.- if (suffixWriter.length() > 2L * numEntries && prefixLength > 2) {- // LZ4 inserts references whenever it sees duplicate strings of 4 chars or more, so only try- // it out if the- // average suffix length is greater than 6.- if (suffixWriter.length() > 6L * numEntries) {- LZ4.compress(- suffixWriter.bytes(), 0, suffixWriter.length(), spareWriter, compressionHashTable);- if (spareWriter.size() < suffixWriter.length() - (suffixWriter.length() >>> 2)) {- // LZ4 saved more than 25%, go for it- compressionAlg = CompressionAlgorithm.LZ4;- }- }- if (compressionAlg == CompressionAlgorithm.NO_COMPRESSION) {- spareWriter.reset();- if (spareBytes.length < suffixWriter.length()) {- spareBytes = new byte[ArrayUtil.oversize(suffixWriter.length(), 1)];- }- if (LowercaseAsciiCompression.compress(- suffixWriter.bytes(), suffixWriter.length(), spareBytes, spareWriter)) {- compressionAlg = CompressionAlgorithm.LOWERCASE_ASCII;- }- }- }- long token = ((long) suffixWriter.length()) << 3;- if (isLeafBlock) {- token |= 0x04;- }- token |= compressionAlg.code;- termsOut.writeVLong(token);- if (compressionAlg == CompressionAlgorithm.NO_COMPRESSION) {- termsOut.writeBytes(suffixWriter.bytes(), suffixWriter.length());- } else {- spareWriter.copyTo(termsOut);- }- suffixWriter.setLength(0);- spareWriter.reset();-- // Write suffix lengths- final int numSuffixBytes = Math.toIntExact(suffixLengthsWriter.size());- spareBytes = ArrayUtil.grow(spareBytes, numSuffixBytes);- suffixLengthsWriter.copyTo(new ByteArrayDataOutput(spareBytes));- suffixLengthsWriter.reset();- if (allEqual(spareBytes, 1, numSuffixBytes, spareBytes[0])) {- // Structured fields like IDs often have most values of the same length- termsOut.writeVInt((numSuffixBytes << 1) | 1);- termsOut.writeByte(spareBytes[0]);- } else {- termsOut.writeVInt(numSuffixBytes << 1);- termsOut.writeBytes(spareBytes, numSuffixBytes);- }-- // Stats- final int numStatsBytes = Math.toIntExact(statsWriter.size());- termsOut.writeVInt(numStatsBytes);- statsWriter.copyTo(termsOut);- statsWriter.reset();-- // Write term meta data byte[] blob- termsOut.writeVInt((int) metaWriter.size());- metaWriter.copyTo(termsOut);- metaWriter.reset();-- // if (DEBUG) {- // System.out.println(" fpEnd=" + out.getFilePointer());- // }-- if (hasFloorLeadLabel) {- // We already allocated to length+1 above:- prefix.bytes[prefix.length++] = (byte) floorLeadLabel;- }-- return new PendingBlock(prefix, startFP, hasTerms, isFloor, floorLeadLabel, subIndices);- }-- TermsWriter(FieldInfo fieldInfo) {- this.fieldInfo = fieldInfo;- assert fieldInfo.getIndexOptions() != IndexOptions.NONE;- docsSeen = new FixedBitSet(maxDoc);- postingsWriter.setField(fieldInfo);- }-- /** Writes one term's worth of postings. */- public void write(BytesRef text, TermsEnum termsEnum, NormsProducer norms) throws IOException {- /*- if (DEBUG) {- int[] tmp = new int[lastTerm.length];- System.arraycopy(prefixStarts, 0, tmp, 0, tmp.length);- System.out.println("BTTW: write term=" + brToString(text) + " prefixStarts=" + Arrays.toString(tmp) + " pending.size()=" + pending.size());- }- */-- BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen, norms);- if (state != null) {-- assert state.docFreq != 0;- assert fieldInfo.getIndexOptions() == IndexOptions.DOCS- || state.totalTermFreq >= state.docFreq- : "postingsWriter=" + postingsWriter;- pushTerm(text);-- PendingTerm term = new PendingTerm(text, state);- pending.add(term);- // if (DEBUG) System.out.println(" add pending term = " + text + " pending.size()=" +- // pending.size());-- sumDocFreq += state.docFreq;- sumTotalTermFreq += state.totalTermFreq;- numTerms++;- if (firstPendingTerm == null) {- firstPendingTerm = term;- }- lastPendingTerm = term;- }- }-- /** Pushes the new term to the top of the stack, and writes new blocks. */- private void pushTerm(BytesRef text) throws IOException {- // Find common prefix between last term and current term:- int prefixLength =- Arrays.mismatch(- lastTerm.bytes(),- 0,- lastTerm.length(),- text.bytes,- text.offset,- text.offset + text.length);- if (prefixLength == -1) { // Only happens for the first term, if it is empty- assert lastTerm.length() == 0;- prefixLength = 0;- }-- // if (DEBUG) System.out.println(" shared=" + pos + " lastTerm.length=" + lastTerm.length);-- // Close the "abandoned" suffix now:- for (int i = lastTerm.length() - 1; i >= prefixLength; i--) {-- // How many items on top of the stack share the current suffix- // we are closing:- int prefixTopSize = pending.size() - prefixStarts[i];- if (prefixTopSize >= minItemsInBlock) {- // if (DEBUG) System.out.println("pushTerm i=" + i + " prefixTopSize=" + prefixTopSize + "- // minItemsInBlock=" + minItemsInBlock);- writeBlocks(i + 1, prefixTopSize);- prefixStarts[i] -= prefixTopSize - 1;- }- }-- if (prefixStarts.length < text.length) {- prefixStarts = ArrayUtil.grow(prefixStarts, text.length);- }-- // Init new tail:- for (int i = prefixLength; i < text.length; i++) {- prefixStarts[i] = pending.size();- }-- lastTerm.copyBytes(text);- }-- // Finishes all terms in this field- public void finish() throws IOException {- if (numTerms > 0) {- // if (DEBUG) System.out.println("BTTW: finish prefixStarts=" +- // Arrays.toString(prefixStarts));-- // Add empty term to force closing of all final blocks:- pushTerm(new BytesRef());-- // TODO: if pending.size() is already 1 with a non-zero prefix length- // we can save writing a "degenerate" root block, but we have to- // fix all the places that assume the root block's prefix is the empty string:- pushTerm(new BytesRef());- writeBlocks(0, pending.size());-- // We better have one final "root" block:- assert pending.size() == 1 && !pending.get(0).isTerm- : "pending.size()=" + pending.size() + " pending=" + pending;- final PendingBlock root = (PendingBlock) pending.get(0);- assert root.prefix.length == 0;- assert root.index.getEmptyOutput() != null;-- // Write FST to index- indexStartFP = indexOut.getFilePointer();- root.index.save(indexOut, indexOut);- // System.out.println(" write FST " + indexStartFP + " field=" + fieldInfo.name);-- /*- if (DEBUG) {- final String dotFileName = segment + "_" + fieldInfo.name + ".dot";- Writer w = new OutputStreamWriter(new FileOutputStream(dotFileName));- Util.toDot(root.index, w, false, false);- System.out.println("SAVED to " + dotFileName);- w.close();- }- */- assert firstPendingTerm != null;- BytesRef minTerm = new BytesRef(firstPendingTerm.termBytes);-- assert lastPendingTerm != null;- BytesRef maxTerm = new BytesRef(lastPendingTerm.termBytes);-- fields.add(- new FieldMetaData(- fieldInfo,- ((PendingBlock) pending.get(0)).index.getEmptyOutput(),- numTerms,- indexStartFP,- sumTotalTermFreq,- sumDocFreq,- docsSeen.cardinality(),- minTerm,- maxTerm));- } else {- assert sumTotalTermFreq == 0- || fieldInfo.getIndexOptions() == IndexOptions.DOCS && sumTotalTermFreq == -1;- assert sumDocFreq == 0;- assert docsSeen.cardinality() == 0;- }- }-- private final ByteBuffersDataOutput suffixLengthsWriter =- ByteBuffersDataOutput.newResettableInstance();- private final BytesRefBuilder suffixWriter = new BytesRefBuilder();- private final ByteBuffersDataOutput statsWriter = ByteBuffersDataOutput.newResettableInstance();- private final ByteBuffersDataOutput metaWriter = ByteBuffersDataOutput.newResettableInstance();- private final ByteBuffersDataOutput spareWriter = ByteBuffersDataOutput.newResettableInstance();- private byte[] spareBytes = BytesRef.EMPTY_BYTES;- private final LZ4.HighCompressionHashTable compressionHashTable =- new LZ4.HighCompressionHashTable();- }-- private boolean closed;-- @Override- public void close() throws IOException {- if (closed) {- return;- }- closed = true;-- boolean success = false;- try {-- final long dirStart = termsOut.getFilePointer();- final long indexDirStart = indexOut.getFilePointer();-- termsOut.writeVInt(fields.size());-- for (FieldMetaData field : fields) {- // System.out.println(" field " + field.fieldInfo.name + " " + field.numTerms + " terms");- termsOut.writeVInt(field.fieldInfo.number);- assert field.numTerms > 0;- termsOut.writeVLong(field.numTerms);- termsOut.writeVInt(field.rootCode.length);- termsOut.writeBytes(field.rootCode.bytes, field.rootCode.offset, field.rootCode.length);- assert field.fieldInfo.getIndexOptions() != IndexOptions.NONE;- if (field.fieldInfo.getIndexOptions() != IndexOptions.DOCS) {- termsOut.writeVLong(field.sumTotalTermFreq);- }- termsOut.writeVLong(field.sumDocFreq);- termsOut.writeVInt(field.docCount);- indexOut.writeVLong(field.indexStartFP);- writeBytesRef(termsOut, field.minTerm);- writeBytesRef(termsOut, field.maxTerm);- }- writeTrailer(termsOut, dirStart);- CodecUtil.writeFooter(termsOut);- writeIndexTrailer(indexOut, indexDirStart);- CodecUtil.writeFooter(indexOut);- success = true;- } finally {- if (success) {- IOUtils.close(termsOut, indexOut, postingsWriter);- } else {- IOUtils.closeWhileHandlingException(termsOut, indexOut, postingsWriter);- }- }- }-- private static void writeBytesRef(IndexOutput out, BytesRef bytes) throws IOException {- out.writeVInt(bytes.length);- out.writeBytes(bytes.bytes, bytes.offset, bytes.length);- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50PostingsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50PostingsWriter.javadeleted file mode 100644index ad44548942b..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50PostingsWriter.java+++ /dev/null@@ -1,529 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import static org.apache.lucene.backward_codecs.lucene50.ForUtil.MAX_DATA_SIZE;-import static org.apache.lucene.backward_codecs.lucene50.ForUtil.MAX_ENCODED_SIZE;-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.BLOCK_SIZE;-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.DOC_CODEC;-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.MAX_SKIP_LEVELS;-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.PAY_CODEC;-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.POS_CODEC;-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.TERMS_CODEC;-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.VERSION_CURRENT;--import java.io.IOException;-import org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.IntBlockTermState;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.BlockTermState;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.CompetitiveImpactAccumulator;-import org.apache.lucene.codecs.PushPostingsWriterBase;-import org.apache.lucene.index.CorruptIndexException;-import org.apache.lucene.index.FieldInfo;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.IndexWriter;-import org.apache.lucene.index.NumericDocValues;-import org.apache.lucene.index.SegmentWriteState;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.store.IndexOutput;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.IOUtils;-import org.apache.lucene.util.packed.PackedInts;--/**- * Concrete class that writes docId(maybe frq,pos,offset,payloads) list with postings format.- *- * <p>Postings list for each term will be stored separately.- *- * @see Lucene50SkipWriter for details about skipping setting and postings layout.- * @lucene.experimental- */-public final class Lucene50PostingsWriter extends PushPostingsWriterBase {-- IndexOutput docOut;- IndexOutput posOut;- IndexOutput payOut;-- static final IntBlockTermState emptyState = new IntBlockTermState();- IntBlockTermState lastState;-- // Holds starting file pointers for current term:- private long docStartFP;- private long posStartFP;- private long payStartFP;-- final int[] docDeltaBuffer;- final int[] freqBuffer;- private int docBufferUpto;-- final int[] posDeltaBuffer;- final int[] payloadLengthBuffer;- final int[] offsetStartDeltaBuffer;- final int[] offsetLengthBuffer;- private int posBufferUpto;-- private byte[] payloadBytes;- private int payloadByteUpto;-- private int lastBlockDocID;- private long lastBlockPosFP;- private long lastBlockPayFP;- private int lastBlockPosBufferUpto;- private int lastBlockPayloadByteUpto;-- private int lastDocID;- private int lastPosition;- private int lastStartOffset;- private int docCount;-- final byte[] encoded;-- private final ForUtil forUtil;- private final Lucene50SkipWriter skipWriter;-- private boolean fieldHasNorms;- private NumericDocValues norms;- private final CompetitiveImpactAccumulator competitiveFreqNormAccumulator =- new CompetitiveImpactAccumulator();-- /** Creates a postings writer */- public Lucene50PostingsWriter(SegmentWriteState state) throws IOException {- final float acceptableOverheadRatio = PackedInts.COMPACT;-- String docFileName =- IndexFileNames.segmentFileName(- state.segmentInfo.name, state.segmentSuffix, Lucene50PostingsFormat.DOC_EXTENSION);- docOut = EndiannessReverserUtil.createOutput(state.directory, docFileName, state.context);- IndexOutput posOut = null;- IndexOutput payOut = null;- boolean success = false;- try {- CodecUtil.writeIndexHeader(- docOut, DOC_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);- forUtil = new ForUtil(acceptableOverheadRatio, docOut);- if (state.fieldInfos.hasProx()) {- posDeltaBuffer = new int[MAX_DATA_SIZE];- String posFileName =- IndexFileNames.segmentFileName(- state.segmentInfo.name, state.segmentSuffix, Lucene50PostingsFormat.POS_EXTENSION);- posOut = EndiannessReverserUtil.createOutput(state.directory, posFileName, state.context);- CodecUtil.writeIndexHeader(- posOut, POS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);-- if (state.fieldInfos.hasPayloads()) {- payloadBytes = new byte[128];- payloadLengthBuffer = new int[MAX_DATA_SIZE];- } else {- payloadBytes = null;- payloadLengthBuffer = null;- }-- if (state.fieldInfos.hasOffsets()) {- offsetStartDeltaBuffer = new int[MAX_DATA_SIZE];- offsetLengthBuffer = new int[MAX_DATA_SIZE];- } else {- offsetStartDeltaBuffer = null;- offsetLengthBuffer = null;- }-- if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {- String payFileName =- IndexFileNames.segmentFileName(- state.segmentInfo.name,- state.segmentSuffix,- Lucene50PostingsFormat.PAY_EXTENSION);- payOut = EndiannessReverserUtil.createOutput(state.directory, payFileName, state.context);- CodecUtil.writeIndexHeader(- payOut, PAY_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);- }- } else {- posDeltaBuffer = null;- payloadLengthBuffer = null;- offsetStartDeltaBuffer = null;- offsetLengthBuffer = null;- payloadBytes = null;- }- this.payOut = payOut;- this.posOut = posOut;- success = true;- } finally {- if (!success) {- IOUtils.closeWhileHandlingException(docOut, posOut, payOut);- }- }-- docDeltaBuffer = new int[MAX_DATA_SIZE];- freqBuffer = new int[MAX_DATA_SIZE];-- // TODO: should we try skipping every 2/4 blocks...?- skipWriter =- new Lucene50SkipWriter(- MAX_SKIP_LEVELS, BLOCK_SIZE, state.segmentInfo.maxDoc(), docOut, posOut, payOut);-- encoded = new byte[MAX_ENCODED_SIZE];- }-- @Override- public IntBlockTermState newTermState() {- return new IntBlockTermState();- }-- @Override- public void init(IndexOutput termsOut, SegmentWriteState state) throws IOException {- CodecUtil.writeIndexHeader(- termsOut, TERMS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);- termsOut.writeVInt(BLOCK_SIZE);- }-- @Override- public void setField(FieldInfo fieldInfo) {- super.setField(fieldInfo);- skipWriter.setField(writePositions, writeOffsets, writePayloads);- lastState = emptyState;- fieldHasNorms = fieldInfo.hasNorms();- }-- @Override- public void startTerm(NumericDocValues norms) {- docStartFP = docOut.getFilePointer();- if (writePositions) {- posStartFP = posOut.getFilePointer();- if (writePayloads || writeOffsets) {- payStartFP = payOut.getFilePointer();- }- }- lastDocID = 0;- lastBlockDocID = -1;- skipWriter.resetSkip();- this.norms = norms;- competitiveFreqNormAccumulator.clear();- }-- @Override- public void startDoc(int docID, int termDocFreq) throws IOException {- // Have collected a block of docs, and get a new doc.- // Should write skip data as well as postings list for- // current block.- if (lastBlockDocID != -1 && docBufferUpto == 0) {- skipWriter.bufferSkip(- lastBlockDocID,- competitiveFreqNormAccumulator,- docCount,- lastBlockPosFP,- lastBlockPayFP,- lastBlockPosBufferUpto,- lastBlockPayloadByteUpto);- competitiveFreqNormAccumulator.clear();- }-- final int docDelta = docID - lastDocID;-- if (docID < 0 || (docCount > 0 && docDelta <= 0)) {- throw new CorruptIndexException(- "docs out of order (" + docID + " <= " + lastDocID + " )", docOut);- }-- docDeltaBuffer[docBufferUpto] = docDelta;- if (writeFreqs) {- freqBuffer[docBufferUpto] = termDocFreq;- }-- docBufferUpto++;- docCount++;-- if (docBufferUpto == BLOCK_SIZE) {- forUtil.writeBlock(docDeltaBuffer, encoded, docOut);- if (writeFreqs) {- forUtil.writeBlock(freqBuffer, encoded, docOut);- }- // NOTE: don't set docBufferUpto back to 0 here;- // finishDoc will do so (because it needs to see that- // the block was filled so it can save skip data)- }-- lastDocID = docID;- lastPosition = 0;- lastStartOffset = 0;-- long norm;- if (fieldHasNorms) {- boolean found = norms.advanceExact(docID);- if (found == false) {- // This can happen if indexing hits a problem after adding a doc to the- // postings but before buffering the norm. Such documents are written- // deleted and will go away on the first merge.- norm = 1L;- } else {- norm = norms.longValue();- assert norm != 0 : docID;- }- } else {- norm = 1L;- }-- competitiveFreqNormAccumulator.add(writeFreqs ? termDocFreq : 1, norm);- }-- @Override- public void addPosition(int position, BytesRef payload, int startOffset, int endOffset)- throws IOException {- if (position > IndexWriter.MAX_POSITION) {- throw new CorruptIndexException(- "position="- + position- + " is too large (> IndexWriter.MAX_POSITION="- + IndexWriter.MAX_POSITION- + ")",- docOut);- }- if (position < 0) {- throw new CorruptIndexException("position=" + position + " is < 0", docOut);- }- posDeltaBuffer[posBufferUpto] = position - lastPosition;- if (writePayloads) {- if (payload == null || payload.length == 0) {- // no payload- payloadLengthBuffer[posBufferUpto] = 0;- } else {- payloadLengthBuffer[posBufferUpto] = payload.length;- if (payloadByteUpto + payload.length > payloadBytes.length) {- payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payload.length);- }- System.arraycopy(- payload.bytes, payload.offset, payloadBytes, payloadByteUpto, payload.length);- payloadByteUpto += payload.length;- }- }-- if (writeOffsets) {- assert startOffset >= lastStartOffset;- assert endOffset >= startOffset;- offsetStartDeltaBuffer[posBufferUpto] = startOffset - lastStartOffset;- offsetLengthBuffer[posBufferUpto] = endOffset - startOffset;- lastStartOffset = startOffset;- }-- posBufferUpto++;- lastPosition = position;- if (posBufferUpto == BLOCK_SIZE) {- forUtil.writeBlock(posDeltaBuffer, encoded, posOut);-- if (writePayloads) {- forUtil.writeBlock(payloadLengthBuffer, encoded, payOut);- payOut.writeVInt(payloadByteUpto);- payOut.writeBytes(payloadBytes, 0, payloadByteUpto);- payloadByteUpto = 0;- }- if (writeOffsets) {- forUtil.writeBlock(offsetStartDeltaBuffer, encoded, payOut);- forUtil.writeBlock(offsetLengthBuffer, encoded, payOut);- }- posBufferUpto = 0;- }- }-- @Override- public void finishDoc() throws IOException {- // Since we don't know df for current term, we had to buffer- // those skip data for each block, and when a new doc comes,- // write them to skip file.- if (docBufferUpto == BLOCK_SIZE) {- lastBlockDocID = lastDocID;- if (posOut != null) {- if (payOut != null) {- lastBlockPayFP = payOut.getFilePointer();- }- lastBlockPosFP = posOut.getFilePointer();- lastBlockPosBufferUpto = posBufferUpto;- lastBlockPayloadByteUpto = payloadByteUpto;- }- docBufferUpto = 0;- }- }-- /** Called when we are done adding docs to this term */- @Override- public void finishTerm(BlockTermState _state) throws IOException {- IntBlockTermState state = (IntBlockTermState) _state;- assert state.docFreq > 0;-- // TODO: wasteful we are counting this (counting # docs- // for this term) in two places?- assert state.docFreq == docCount : state.docFreq + " vs " + docCount;-- // docFreq == 1, don't write the single docid/freq to a separate file along with a pointer to- // it.- final int singletonDocID;- if (state.docFreq == 1) {- // pulse the singleton docid into the term dictionary, freq is implicitly totalTermFreq- singletonDocID = docDeltaBuffer[0];- } else {- singletonDocID = -1;- // vInt encode the remaining doc deltas and freqs:- for (int i = 0; i < docBufferUpto; i++) {- final int docDelta = docDeltaBuffer[i];- final int freq = freqBuffer[i];- if (!writeFreqs) {- docOut.writeVInt(docDelta);- } else if (freqBuffer[i] == 1) {- docOut.writeVInt((docDelta << 1) | 1);- } else {- docOut.writeVInt(docDelta << 1);- docOut.writeVInt(freq);- }- }- }-- final long lastPosBlockOffset;-- if (writePositions) {- // totalTermFreq is just total number of positions(or payloads, or offsets)- // associated with current term.- assert state.totalTermFreq != -1;- if (state.totalTermFreq > BLOCK_SIZE) {- // record file offset for last pos in last block- lastPosBlockOffset = posOut.getFilePointer() - posStartFP;- } else {- lastPosBlockOffset = -1;- }- if (posBufferUpto > 0) {- // TODO: should we send offsets/payloads to- // .pay...? seems wasteful (have to store extra- // vLong for low (< BLOCK_SIZE) DF terms = vast vast- // majority)-- // vInt encode the remaining positions/payloads/offsets:- int lastPayloadLength = -1; // force first payload length to be written- int lastOffsetLength = -1; // force first offset length to be written- int payloadBytesReadUpto = 0;- for (int i = 0; i < posBufferUpto; i++) {- final int posDelta = posDeltaBuffer[i];- if (writePayloads) {- final int payloadLength = payloadLengthBuffer[i];- if (payloadLength != lastPayloadLength) {- lastPayloadLength = payloadLength;- posOut.writeVInt((posDelta << 1) | 1);- posOut.writeVInt(payloadLength);- } else {- posOut.writeVInt(posDelta << 1);- }-- if (payloadLength != 0) {- posOut.writeBytes(payloadBytes, payloadBytesReadUpto, payloadLength);- payloadBytesReadUpto += payloadLength;- }- } else {- posOut.writeVInt(posDelta);- }-- if (writeOffsets) {- int delta = offsetStartDeltaBuffer[i];- int length = offsetLengthBuffer[i];- if (length == lastOffsetLength) {- posOut.writeVInt(delta << 1);- } else {- posOut.writeVInt(delta << 1 | 1);- posOut.writeVInt(length);- lastOffsetLength = length;- }- }- }-- if (writePayloads) {- assert payloadBytesReadUpto == payloadByteUpto;- payloadByteUpto = 0;- }- }- } else {- lastPosBlockOffset = -1;- }-- long skipOffset;- if (docCount > BLOCK_SIZE) {- skipOffset = skipWriter.writeSkip(docOut) - docStartFP;- } else {- skipOffset = -1;- }-- state.docStartFP = docStartFP;- state.posStartFP = posStartFP;- state.payStartFP = payStartFP;- state.singletonDocID = singletonDocID;- state.skipOffset = skipOffset;- state.lastPosBlockOffset = lastPosBlockOffset;- docBufferUpto = 0;- posBufferUpto = 0;- lastDocID = 0;- docCount = 0;- }-- @Override- public void encodeTerm(- DataOutput out, FieldInfo fieldInfo, BlockTermState _state, boolean absolute)- throws IOException {- IntBlockTermState state = (IntBlockTermState) _state;- if (absolute) {- lastState = emptyState;- }- out.writeVLong(state.docStartFP - lastState.docStartFP);- if (writePositions) {- out.writeVLong(state.posStartFP - lastState.posStartFP);- if (writePayloads || writeOffsets) {- out.writeVLong(state.payStartFP - lastState.payStartFP);- }- }- if (state.singletonDocID != -1) {- out.writeVInt(state.singletonDocID);- }- if (writePositions) {- if (state.lastPosBlockOffset != -1) {- out.writeVLong(state.lastPosBlockOffset);- }- }- if (state.skipOffset != -1) {- out.writeVLong(state.skipOffset);- }- lastState = state;- }-- @Override- public void close() throws IOException {- // TODO: add a finish() at least to PushBase? DV too...?- boolean success = false;- try {- if (docOut != null) {- CodecUtil.writeFooter(docOut);- }- if (posOut != null) {- CodecUtil.writeFooter(posOut);- }- if (payOut != null) {- CodecUtil.writeFooter(payOut);- }- success = true;- } finally {- if (success) {- IOUtils.close(docOut, posOut, payOut);- } else {- IOUtils.closeWhileHandlingException(docOut, posOut, payOut);- }- docOut = posOut = payOut = null;- }- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWCompoundFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWCompoundFormat.javadeleted file mode 100644index cfd269d9ddd..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWCompoundFormat.java+++ /dev/null@@ -1,142 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import java.io.IOException;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.CompoundDirectory;-import org.apache.lucene.codecs.CompoundFormat;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.SegmentInfo;-import org.apache.lucene.store.ChecksumIndexInput;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IndexOutput;--/**- * Lucene 5.0 compound file format- *- * <p>Files:- *- * <ul>- * <li><code>.cfs</code>: An optional "virtual" file consisting of all the other index files for- * systems that frequently run out of file handles.- * <li><code>.cfe</code>: The "virtual" compound file's entry table holding all entries in the- * corresponding .cfs file.- * </ul>- *- * <p>Description:- *- * <ul>- * <li>Compound (.cfs) --> Header, FileData <sup>FileCount</sup>, Footer- * <li>Compound Entry Table (.cfe) --> Header, FileCount, <FileName, DataOffset,- * DataLength> <sup>FileCount</sup>- * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}- * <li>FileCount --> {@link DataOutput#writeVInt VInt}- * <li>DataOffset,DataLength,Checksum --> {@link DataOutput#writeLong UInt64}- * <li>FileName --> {@link DataOutput#writeString String}- * <li>FileData --> raw file data- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}- * </ul>- *- * <p>Notes:- *- * <ul>- * <li>FileCount indicates how many files are contained in this compound file. The entry table- * that follows has that many entries.- * <li>Each directory entry contains a long pointer to the start of this file's data section, the- * files length, and a String with that file's name.- * </ul>- */-public final class Lucene50RWCompoundFormat extends CompoundFormat {-- /** Extension of compound file */- static final String DATA_EXTENSION = "cfs";-- /** Extension of compound file entries */- static final String ENTRIES_EXTENSION = "cfe";-- static final String DATA_CODEC = "Lucene50CompoundData";- static final String ENTRY_CODEC = "Lucene50CompoundEntries";- static final int VERSION_START = 0;- static final int VERSION_CURRENT = VERSION_START;-- /** Sole constructor. */- public Lucene50RWCompoundFormat() {}-- @Override- public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si) throws IOException {- return new Lucene50CompoundReader(dir, si);- }-- @Override- public void write(Directory dir, SegmentInfo si, IOContext context) throws IOException {- String dataFile = IndexFileNames.segmentFileName(si.name, "", DATA_EXTENSION);- String entriesFile = IndexFileNames.segmentFileName(si.name, "", ENTRIES_EXTENSION);-- try (IndexOutput data = EndiannessReverserUtil.createOutput(dir, dataFile, context);- IndexOutput entries = EndiannessReverserUtil.createOutput(dir, entriesFile, context)) {- CodecUtil.writeIndexHeader(data, DATA_CODEC, VERSION_CURRENT, si.getId(), "");- CodecUtil.writeIndexHeader(entries, ENTRY_CODEC, VERSION_CURRENT, si.getId(), "");-- writeCompoundFile(entries, data, dir, si);-- CodecUtil.writeFooter(data);- CodecUtil.writeFooter(entries);- }- }-- private void writeCompoundFile(- IndexOutput entries, IndexOutput data, Directory dir, SegmentInfo si) throws IOException {- // write number of files- entries.writeVInt(si.files().size());- for (String file : si.files()) {- // write bytes for file- long startOffset = data.getFilePointer();- try (ChecksumIndexInput in =- EndiannessReverserUtil.openChecksumInput(dir, file, IOContext.READONCE)) {-- // just copies the index header, verifying that its id matches what we expect- CodecUtil.verifyAndCopyIndexHeader(in, data, si.getId());-- // copy all bytes except the footer- long numBytesToCopy = in.length() - CodecUtil.footerLength() - in.getFilePointer();- data.copyBytes(in, numBytesToCopy);-- // verify footer (checksum) matches for the incoming file we are copying- long checksum = CodecUtil.checkFooter(in);-- // this is poached from CodecUtil.writeFooter, but we need to use our own checksum, not- // data.getChecksum(), but I think- // adding a public method to CodecUtil to do that is somewhat dangerous:- CodecUtil.writeBEInt(data, CodecUtil.FOOTER_MAGIC);- CodecUtil.writeBEInt(data, 0);- CodecUtil.writeBELong(data, checksum);- }- long endOffset = data.getFilePointer();-- long length = endOffset - startOffset;-- // write entry for file- entries.writeString(IndexFileNames.stripSegmentName(file));- entries.writeLong(startOffset);- entries.writeLong(length);- }- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWPostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWPostingsFormat.javadeleted file mode 100644index fb042a1a9ad..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWPostingsFormat.java+++ /dev/null@@ -1,47 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import java.io.IOException;-import org.apache.lucene.backward_codecs.lucene40.blocktree.Lucene40BlockTreeTermsWriterV5;-import org.apache.lucene.codecs.FieldsConsumer;-import org.apache.lucene.codecs.PostingsWriterBase;-import org.apache.lucene.index.SegmentWriteState;-import org.apache.lucene.util.IOUtils;--public class Lucene50RWPostingsFormat extends Lucene50PostingsFormat {-- @Override- public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {- PostingsWriterBase postingsWriter = new Lucene50PostingsWriter(state);- boolean success = false;- try {- FieldsConsumer ret =- new Lucene40BlockTreeTermsWriterV5(- state,- postingsWriter,- Lucene40BlockTreeTermsWriterV5.DEFAULT_MIN_BLOCK_SIZE,- Lucene40BlockTreeTermsWriterV5.DEFAULT_MAX_BLOCK_SIZE);- success = true;- return ret;- } finally {- if (!success) {- IOUtils.closeWhileHandlingException(postingsWriter);- }- }- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWStoredFieldsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWStoredFieldsFormat.javadeleted file mode 100644index 320819ae604..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWStoredFieldsFormat.java+++ /dev/null@@ -1,72 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import java.io.IOException;-import org.apache.lucene.backward_codecs.compressing.CompressionMode;-import org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50RWCompressingStoredFieldsFormat;-import org.apache.lucene.codecs.StoredFieldsFormat;-import org.apache.lucene.codecs.StoredFieldsWriter;-import org.apache.lucene.index.SegmentInfo;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;--/** RW impersonation of Lucene50StoredFieldsFormat. */-public final class Lucene50RWStoredFieldsFormat extends Lucene50StoredFieldsFormat {-- /** No-argument constructor. */- public Lucene50RWStoredFieldsFormat() {- super();- }-- /** Constructor that takes a mode. */- public Lucene50RWStoredFieldsFormat(Lucene50StoredFieldsFormat.Mode mode) {- super(mode);- }-- @Override- public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context)- throws IOException {- String previous = si.putAttribute(MODE_KEY, mode.name());- if (previous != null && previous.equals(mode.name()) == false) {- throw new IllegalStateException(- "found existing value for "- + MODE_KEY- + " for segment: "- + si.name- + "old="- + previous- + ", new="- + mode.name());- }- return impl(mode).fieldsWriter(directory, si, context);- }-- @Override- StoredFieldsFormat impl(Mode mode) {- switch (mode) {- case BEST_SPEED:- return new Lucene50RWCompressingStoredFieldsFormat(- "Lucene50StoredFieldsFastData", Lucene50StoredFieldsFormat.FAST_MODE, 1 << 14, 128, 10);- case BEST_COMPRESSION:- return new Lucene50RWCompressingStoredFieldsFormat(- "Lucene50StoredFieldsHighData", CompressionMode.HIGH_COMPRESSION, 61440, 512, 10);- default:- throw new AssertionError();- }- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWTermVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWTermVectorsFormat.javadeleted file mode 100644index d711a3f267c..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWTermVectorsFormat.java+++ /dev/null@@ -1,28 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50RWCompressingTermVectorsFormat;--/** RW impersonation of Lucene50StoredFieldsFormat. */-public final class Lucene50RWTermVectorsFormat extends Lucene50RWCompressingTermVectorsFormat {-- /** Sole constructor. */- public Lucene50RWTermVectorsFormat() {- super("Lucene50TermVectorsData", "", Lucene50StoredFieldsFormat.FAST_MODE, 1 << 12, 128, 10);- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50SkipWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50SkipWriter.javadeleted file mode 100644index f9b191c1e6f..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50SkipWriter.java+++ /dev/null@@ -1,237 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import java.io.IOException;-import java.util.Arrays;-import java.util.Collection;-import org.apache.lucene.codecs.CompetitiveImpactAccumulator;-import org.apache.lucene.codecs.Impact;-import org.apache.lucene.codecs.MultiLevelSkipListWriter;-import org.apache.lucene.store.ByteBuffersDataOutput;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.store.IndexOutput;--/**- * Write skip lists with multiple levels, and support skip within block ints.- *- * <p>Assume that docFreq = 28, skipInterval = blockSize = 12- *- * <pre>- * | block#0 | | block#1 | |vInts|- * d d d d d d d d d d d d d d d d d d d d d d d d d d d d (posting list)- * ^ ^ (level 0 skip point)- * </pre>- *- * Note that skipWriter will ignore first document in block#0, since it is useless as a skip point.- * Also, we'll never skip into the vInts block, only record skip data at the start its start- * point(if it exist).- *- * <p>For each skip point, we will record: 1. docID in former position, i.e. for position 12, record- * docID[11], etc. 2. its related file points(position, payload), 3. related numbers or- * uptos(position, payload). 4. start offset.- */-final class Lucene50SkipWriter extends MultiLevelSkipListWriter {- private int[] lastSkipDoc;- private long[] lastSkipDocPointer;- private long[] lastSkipPosPointer;- private long[] lastSkipPayPointer;- private int[] lastPayloadByteUpto;-- private final IndexOutput docOut;- private final IndexOutput posOut;- private final IndexOutput payOut;-- private int curDoc;- private long curDocPointer;- private long curPosPointer;- private long curPayPointer;- private int curPosBufferUpto;- private int curPayloadByteUpto;- private CompetitiveImpactAccumulator[] curCompetitiveFreqNorms;- private boolean fieldHasPositions;- private boolean fieldHasOffsets;- private boolean fieldHasPayloads;-- public Lucene50SkipWriter(- int maxSkipLevels,- int blockSize,- int docCount,- IndexOutput docOut,- IndexOutput posOut,- IndexOutput payOut) {- super(blockSize, 8, maxSkipLevels, docCount);- this.docOut = docOut;- this.posOut = posOut;- this.payOut = payOut;-- lastSkipDoc = new int[maxSkipLevels];- lastSkipDocPointer = new long[maxSkipLevels];- if (posOut != null) {- lastSkipPosPointer = new long[maxSkipLevels];- if (payOut != null) {- lastSkipPayPointer = new long[maxSkipLevels];- }- lastPayloadByteUpto = new int[maxSkipLevels];- }- curCompetitiveFreqNorms = new CompetitiveImpactAccumulator[maxSkipLevels];- for (int i = 0; i < maxSkipLevels; ++i) {- curCompetitiveFreqNorms[i] = new CompetitiveImpactAccumulator();- }- }-- public void setField(- boolean fieldHasPositions, boolean fieldHasOffsets, boolean fieldHasPayloads) {- this.fieldHasPositions = fieldHasPositions;- this.fieldHasOffsets = fieldHasOffsets;- this.fieldHasPayloads = fieldHasPayloads;- }-- // tricky: we only skip data for blocks (terms with more than 128 docs), but re-init'ing the- // skipper- // is pretty slow for rare terms in large segments as we have to fill O(log #docs in segment) of- // junk.- // this is the vast majority of terms (worst case: ID field or similar). so in resetSkip() we- // save- // away the previous pointers, and lazy-init only if we need to buffer skip data for the term.- private boolean initialized;- long lastDocFP;- long lastPosFP;- long lastPayFP;-- @Override- public void resetSkip() {- lastDocFP = docOut.getFilePointer();- if (fieldHasPositions) {- lastPosFP = posOut.getFilePointer();- if (fieldHasOffsets || fieldHasPayloads) {- lastPayFP = payOut.getFilePointer();- }- }- if (initialized) {- for (CompetitiveImpactAccumulator acc : curCompetitiveFreqNorms) {- acc.clear();- }- }- initialized = false;- }-- private void initSkip() {- if (!initialized) {- super.resetSkip();- Arrays.fill(lastSkipDoc, 0);- Arrays.fill(lastSkipDocPointer, lastDocFP);- if (fieldHasPositions) {- Arrays.fill(lastSkipPosPointer, lastPosFP);- if (fieldHasPayloads) {- Arrays.fill(lastPayloadByteUpto, 0);- }- if (fieldHasOffsets || fieldHasPayloads) {- Arrays.fill(lastSkipPayPointer, lastPayFP);- }- }- // sets of competitive freq,norm pairs should be empty at this point- assert Arrays.stream(curCompetitiveFreqNorms)- .map(CompetitiveImpactAccumulator::getCompetitiveFreqNormPairs)- .mapToInt(Collection::size)- .sum()- == 0;- initialized = true;- }- }-- /** Sets the values for the current skip data. */- public void bufferSkip(- int doc,- CompetitiveImpactAccumulator competitiveFreqNorms,- int numDocs,- long posFP,- long payFP,- int posBufferUpto,- int payloadByteUpto)- throws IOException {- initSkip();- this.curDoc = doc;- this.curDocPointer = docOut.getFilePointer();- this.curPosPointer = posFP;- this.curPayPointer = payFP;- this.curPosBufferUpto = posBufferUpto;- this.curPayloadByteUpto = payloadByteUpto;- this.curCompetitiveFreqNorms[0].addAll(competitiveFreqNorms);- bufferSkip(numDocs);- }-- private final ByteBuffersDataOutput freqNormOut = ByteBuffersDataOutput.newResettableInstance();-- @Override- protected void writeSkipData(int level, DataOutput skipBuffer) throws IOException {-- int delta = curDoc - lastSkipDoc[level];-- skipBuffer.writeVInt(delta);- lastSkipDoc[level] = curDoc;-- skipBuffer.writeVLong(curDocPointer - lastSkipDocPointer[level]);- lastSkipDocPointer[level] = curDocPointer;-- if (fieldHasPositions) {-- skipBuffer.writeVLong(curPosPointer - lastSkipPosPointer[level]);- lastSkipPosPointer[level] = curPosPointer;- skipBuffer.writeVInt(curPosBufferUpto);-- if (fieldHasPayloads) {- skipBuffer.writeVInt(curPayloadByteUpto);- }-- if (fieldHasOffsets || fieldHasPayloads) {- skipBuffer.writeVLong(curPayPointer - lastSkipPayPointer[level]);- lastSkipPayPointer[level] = curPayPointer;- }- }-- CompetitiveImpactAccumulator competitiveFreqNorms = curCompetitiveFreqNorms[level];- assert competitiveFreqNorms.getCompetitiveFreqNormPairs().size() > 0;- if (level + 1 < numberOfSkipLevels) {- curCompetitiveFreqNorms[level + 1].addAll(competitiveFreqNorms);- }- writeImpacts(competitiveFreqNorms, freqNormOut);- skipBuffer.writeVInt(Math.toIntExact(freqNormOut.size()));- freqNormOut.copyTo(skipBuffer);- freqNormOut.reset();- competitiveFreqNorms.clear();- }-- static void writeImpacts(CompetitiveImpactAccumulator acc, DataOutput out) throws IOException {- Collection<Impact> impacts = acc.getCompetitiveFreqNormPairs();- Impact previous = new Impact(0, 0);- for (Impact impact : impacts) {- assert impact.freq > previous.freq;- assert Long.compareUnsigned(impact.norm, previous.norm) > 0;- int freqDelta = impact.freq - previous.freq - 1;- long normDelta = impact.norm - previous.norm - 1;- if (normDelta == 0) {- // most of time, norm only increases by 1, so we can fold everything in a single byte- out.writeVInt(freqDelta << 1);- } else {- out.writeVInt((freqDelta << 1) | 1);- out.writeZLong(normDelta);- }- previous = impact;- }- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestBlockPostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestBlockPostingsFormat.javadeleted file mode 100644index 4205b5d70e5..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestBlockPostingsFormat.java+++ /dev/null@@ -1,138 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import java.io.IOException;-import java.util.ArrayList;-import java.util.Arrays;-import java.util.Collections;-import java.util.List;-import org.apache.lucene.backward_codecs.lucene40.blocktree.FieldReader;-import org.apache.lucene.backward_codecs.lucene40.blocktree.Stats;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.Codec;-import org.apache.lucene.codecs.CompetitiveImpactAccumulator;-import org.apache.lucene.codecs.Impact;-import org.apache.lucene.document.Document;-import org.apache.lucene.document.Field;-import org.apache.lucene.index.DirectoryReader;-import org.apache.lucene.index.FreqAndNormBuffer;-import org.apache.lucene.index.IndexWriter;-import org.apache.lucene.index.IndexWriterConfig;-import org.apache.lucene.store.ByteArrayDataInput;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.store.IndexOutput;-import org.apache.lucene.tests.analysis.MockAnalyzer;-import org.apache.lucene.tests.index.BasePostingsFormatTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;-import org.apache.lucene.tests.util.TestUtil;--/** Tests BlockPostingsFormat */-@Nightly // N-2 formats are only tested on nightly runs-public class TestBlockPostingsFormat extends BasePostingsFormatTestCase {- private final Codec codec = TestUtil.alwaysPostingsFormat(new Lucene50RWPostingsFormat());-- @Override- protected Codec getCodec() {- return codec;- }-- /** Make sure the final sub-block(s) are not skipped. */- public void testFinalBlock() throws Exception {- Directory d = newDirectory();- IndexWriter w = new IndexWriter(d, new IndexWriterConfig(new MockAnalyzer(random())));- for (int i = 0; i < 25; i++) {- Document doc = new Document();- doc.add(newStringField("field", Character.toString((char) (97 + i)), Field.Store.NO));- doc.add(newStringField("field", "z" + Character.toString((char) (97 + i)), Field.Store.NO));- w.addDocument(doc);- }- w.forceMerge(1);-- DirectoryReader r = DirectoryReader.open(w);- assertEquals(1, r.leaves().size());- FieldReader field = (FieldReader) r.leaves().get(0).reader().terms("field");- // We should see exactly two blocks: one root block (prefix empty string) and one block for z*- // terms (prefix z):- Stats stats = field.getStats();- assertEquals(0, stats.floorBlockCount);- assertEquals(2, stats.nonFloorBlockCount);- r.close();- w.close();- d.close();- }-- public void testImpactSerialization() throws IOException {- // omit norms and omit freqs- doTestImpactSerialization(Collections.singletonList(new Impact(1, 1L)));-- // omit freqs- doTestImpactSerialization(Collections.singletonList(new Impact(1, 42L)));- // omit freqs with very large norms- doTestImpactSerialization(Collections.singletonList(new Impact(1, -100L)));-- // omit norms- doTestImpactSerialization(Collections.singletonList(new Impact(30, 1L)));- // omit norms with large freq- doTestImpactSerialization(Collections.singletonList(new Impact(500, 1L)));-- // freqs and norms, basic- doTestImpactSerialization(- Arrays.asList(- new Impact(1, 7L),- new Impact(3, 9L),- new Impact(7, 10L),- new Impact(15, 11L),- new Impact(20, 13L),- new Impact(28, 14L)));-- // freqs and norms, high values- doTestImpactSerialization(- Arrays.asList(- new Impact(2, 2L),- new Impact(10, 10L),- new Impact(12, 50L),- new Impact(50, -100L),- new Impact(1000, -80L),- new Impact(1005, -3L)));- }-- private void doTestImpactSerialization(List<Impact> impacts) throws IOException {- CompetitiveImpactAccumulator acc = new CompetitiveImpactAccumulator();- for (Impact impact : impacts) {- acc.add(impact.freq, impact.norm);- }- try (Directory dir = newDirectory()) {- try (IndexOutput out = EndiannessReverserUtil.createOutput(dir, "foo", IOContext.DEFAULT)) {- Lucene50SkipWriter.writeImpacts(acc, out);- }- try (IndexInput in = EndiannessReverserUtil.openInput(dir, "foo", IOContext.DEFAULT)) {- byte[] b = new byte[Math.toIntExact(in.length())];- in.readBytes(b, 0, b.length);- FreqAndNormBuffer impacts2 =- Lucene50ScoreSkipReader.readImpacts(new ByteArrayDataInput(b), new FreqAndNormBuffer());- List<Impact> impacts2AsList = new ArrayList<>();- for (int i = 0; i < impacts2.size; ++i) {- impacts2AsList.add(new Impact(impacts2.freqs[i], impacts2.norms[i]));- }- assertEquals(impacts, impacts2AsList);- }- }- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestBlockPostingsFormat2.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestBlockPostingsFormat2.javadeleted file mode 100644index 337ec450c43..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestBlockPostingsFormat2.java+++ /dev/null@@ -1,135 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import org.apache.lucene.document.Document;-import org.apache.lucene.document.Field;-import org.apache.lucene.document.FieldType;-import org.apache.lucene.document.TextField;-import org.apache.lucene.index.IndexOptions;-import org.apache.lucene.index.IndexWriter;-import org.apache.lucene.index.IndexWriterConfig;-import org.apache.lucene.index.IndexWriterConfig.OpenMode;-import org.apache.lucene.index.IndexableField;-import org.apache.lucene.store.Directory;-import org.apache.lucene.tests.analysis.MockAnalyzer;-import org.apache.lucene.tests.index.RandomIndexWriter;-import org.apache.lucene.tests.util.LuceneTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;-import org.apache.lucene.tests.util.TestUtil;--/** Tests special cases of BlockPostingsFormat */-@Nightly // N-2 formats are only tested on nightly runs-public class TestBlockPostingsFormat2 extends LuceneTestCase {- Directory dir;- RandomIndexWriter iw;-- @Override- public void setUp() throws Exception {- super.setUp();- dir = newFSDirectory(createTempDir("testDFBlockSize"));- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));- iwc.setCodec(TestUtil.alwaysPostingsFormat(new Lucene50RWPostingsFormat()));- iw = new RandomIndexWriter(random(), dir, iwc);- iw.setDoRandomForceMerge(false); // we will ourselves- }-- @Override- public void tearDown() throws Exception {- iw.close();- TestUtil.checkIndex(dir); // for some extra coverage, checkIndex before we forceMerge- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));- iwc.setCodec(TestUtil.alwaysPostingsFormat(new Lucene50RWPostingsFormat()));- iwc.setOpenMode(OpenMode.APPEND);- IndexWriter iw = new IndexWriter(dir, iwc);- iw.forceMerge(1);- iw.close();- dir.close(); // just force a checkindex for now- super.tearDown();- }-- private Document newDocument() {- Document doc = new Document();- for (IndexOptions option : IndexOptions.values()) {- if (option == IndexOptions.NONE) {- continue;- }- FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);- // turn on tvs for a cross-check, since we rely upon checkindex in this test (for now)- ft.setStoreTermVectors(true);- ft.setStoreTermVectorOffsets(true);- ft.setStoreTermVectorPositions(true);- ft.setStoreTermVectorPayloads(true);- ft.setIndexOptions(option);- doc.add(new Field(option.toString(), "", ft));- }- return doc;- }-- /** tests terms with df = blocksize */- public void testDFBlockSize() throws Exception {- Document doc = newDocument();- for (int i = 0; i < Lucene50PostingsFormat.BLOCK_SIZE; i++) {- for (IndexableField f : doc.getFields()) {- ((Field) f).setStringValue(f.name() + " " + f.name() + "_2");- }- iw.addDocument(doc);- }- }-- /** tests terms with df % blocksize = 0 */- public void testDFBlockSizeMultiple() throws Exception {- Document doc = newDocument();- for (int i = 0; i < Lucene50PostingsFormat.BLOCK_SIZE * 16; i++) {- for (IndexableField f : doc.getFields()) {- ((Field) f).setStringValue(f.name() + " " + f.name() + "_2");- }- iw.addDocument(doc);- }- }-- /** tests terms with ttf = blocksize */- public void testTTFBlockSize() throws Exception {- Document doc = newDocument();- for (int i = 0; i < Lucene50PostingsFormat.BLOCK_SIZE / 2; i++) {- for (IndexableField f : doc.getFields()) {- ((Field) f)- .setStringValue(f.name() + " " + f.name() + " " + f.name() + "_2 " + f.name() + "_2");- }- iw.addDocument(doc);- }- }-- /** tests terms with ttf % blocksize = 0 */- public void testTTFBlockSizeMultiple() throws Exception {- Document doc = newDocument();- for (int i = 0; i < Lucene50PostingsFormat.BLOCK_SIZE / 2; i++) {- for (IndexableField f : doc.getFields()) {- String proto =- (f.name() + " " + f.name() + " " + f.name() + " " + f.name() + " " + f.name() + "_2 "- + f.name() + "_2 " + f.name() + "_2 " + f.name() + "_2");- StringBuilder val = new StringBuilder();- for (int j = 0; j < 16; j++) {- val.append(proto);- val.append(" ");- }- ((Field) f).setStringValue(val.toString());- }- iw.addDocument(doc);- }- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestBlockPostingsFormat3.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestBlockPostingsFormat3.javadeleted file mode 100644index 84f6ac950e1..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestBlockPostingsFormat3.java+++ /dev/null@@ -1,493 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import java.util.ArrayList;-import java.util.Collections;-import java.util.HashSet;-import java.util.Random;-import org.apache.lucene.analysis.Analyzer;-import org.apache.lucene.analysis.TokenFilter;-import org.apache.lucene.analysis.Tokenizer;-import org.apache.lucene.document.Document;-import org.apache.lucene.document.Field;-import org.apache.lucene.document.FieldType;-import org.apache.lucene.document.TextField;-import org.apache.lucene.index.DirectoryReader;-import org.apache.lucene.index.IndexOptions;-import org.apache.lucene.index.IndexWriter;-import org.apache.lucene.index.IndexWriterConfig;-import org.apache.lucene.index.IndexWriterConfig.OpenMode;-import org.apache.lucene.index.LeafReader;-import org.apache.lucene.index.LeafReaderContext;-import org.apache.lucene.index.PostingsEnum;-import org.apache.lucene.index.Terms;-import org.apache.lucene.index.TermsEnum;-import org.apache.lucene.index.TermsEnum.SeekStatus;-import org.apache.lucene.search.DocIdSetIterator;-import org.apache.lucene.store.Directory;-import org.apache.lucene.tests.analysis.MockFixedLengthPayloadFilter;-import org.apache.lucene.tests.analysis.MockTokenizer;-import org.apache.lucene.tests.analysis.MockVariableLengthPayloadFilter;-import org.apache.lucene.tests.index.RandomIndexWriter;-import org.apache.lucene.tests.util.English;-import org.apache.lucene.tests.util.LuceneTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;-import org.apache.lucene.tests.util.TestUtil;-import org.apache.lucene.tests.util.automaton.AutomatonTestUtil;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.automaton.Automaton;-import org.apache.lucene.util.automaton.CompiledAutomaton;-import org.apache.lucene.util.automaton.Operations;-import org.apache.lucene.util.automaton.RegExp;--/** Tests partial enumeration (only pulling a subset of the indexed data) */-@Nightly // N-2 formats are only tested on nightly runs-public class TestBlockPostingsFormat3 extends LuceneTestCase {- private final int MAXDOC =- TEST_NIGHTLY ? Lucene50PostingsFormat.BLOCK_SIZE * 20 : Lucene50PostingsFormat.BLOCK_SIZE * 3;-- // creates 8 fields with different options and does "duels" of fields against each other- public void test() throws Exception {- Directory dir = newDirectory();- Analyzer analyzer =- new Analyzer(Analyzer.PER_FIELD_REUSE_STRATEGY) {- @Override- protected TokenStreamComponents createComponents(String fieldName) {- Tokenizer tokenizer = new MockTokenizer();- if (fieldName.contains("payloadsFixed")) {- TokenFilter filter = new MockFixedLengthPayloadFilter(new Random(0), tokenizer, 1);- return new TokenStreamComponents(tokenizer, filter);- } else if (fieldName.contains("payloadsVariable")) {- TokenFilter filter = new MockVariableLengthPayloadFilter(new Random(0), tokenizer);- return new TokenStreamComponents(tokenizer, filter);- } else {- return new TokenStreamComponents(tokenizer);- }- }- };- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);- iwc.setCodec(TestUtil.alwaysPostingsFormat(new Lucene50RWPostingsFormat()));- // TODO we could actually add more fields implemented with different PFs- // or, just put this test into the usual rotation?- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);- Document doc = new Document();- FieldType docsOnlyType = new FieldType(TextField.TYPE_NOT_STORED);- // turn this on for a cross-check- docsOnlyType.setStoreTermVectors(true);- docsOnlyType.setIndexOptions(IndexOptions.DOCS);-- FieldType docsAndFreqsType = new FieldType(TextField.TYPE_NOT_STORED);- // turn this on for a cross-check- docsAndFreqsType.setStoreTermVectors(true);- docsAndFreqsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS);-- FieldType positionsType = new FieldType(TextField.TYPE_NOT_STORED);- // turn these on for a cross-check- positionsType.setStoreTermVectors(true);- positionsType.setStoreTermVectorPositions(true);- positionsType.setStoreTermVectorOffsets(true);- positionsType.setStoreTermVectorPayloads(true);- FieldType offsetsType = new FieldType(positionsType);- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);- Field field1 = new Field("field1docs", "", docsOnlyType);- Field field2 = new Field("field2freqs", "", docsAndFreqsType);- Field field3 = new Field("field3positions", "", positionsType);- Field field4 = new Field("field4offsets", "", offsetsType);- Field field5 = new Field("field5payloadsFixed", "", positionsType);- Field field6 = new Field("field6payloadsVariable", "", positionsType);- Field field7 = new Field("field7payloadsFixedOffsets", "", offsetsType);- Field field8 = new Field("field8payloadsVariableOffsets", "", offsetsType);- doc.add(field1);- doc.add(field2);- doc.add(field3);- doc.add(field4);- doc.add(field5);- doc.add(field6);- doc.add(field7);- doc.add(field8);- for (int i = 0; i < MAXDOC; i++) {- String stringValue =- Integer.toString(i)- + " verycommon "- + English.intToEnglish(i).replace('-', ' ')- + " "- + TestUtil.randomSimpleString(random());- field1.setStringValue(stringValue);- field2.setStringValue(stringValue);- field3.setStringValue(stringValue);- field4.setStringValue(stringValue);- field5.setStringValue(stringValue);- field6.setStringValue(stringValue);- field7.setStringValue(stringValue);- field8.setStringValue(stringValue);- iw.addDocument(doc);- }- iw.close();- verify(dir);- TestUtil.checkIndex(dir); // for some extra coverage, checkIndex before we forceMerge- iwc = newIndexWriterConfig(analyzer);- iwc.setCodec(TestUtil.alwaysPostingsFormat(new Lucene50RWPostingsFormat()));- iwc.setOpenMode(OpenMode.APPEND);- IndexWriter iw2 = new IndexWriter(dir, iwc);- iw2.forceMerge(1);- iw2.close();- verify(dir);- dir.close();- }-- private void verify(Directory dir) throws Exception {- DirectoryReader ir = DirectoryReader.open(dir);- for (LeafReaderContext leaf : ir.leaves()) {- LeafReader leafReader = leaf.reader();- assertTerms(leafReader.terms("field1docs"), leafReader.terms("field2freqs"), true);- assertTerms(leafReader.terms("field3positions"), leafReader.terms("field4offsets"), true);- assertTerms(leafReader.terms("field4offsets"), leafReader.terms("field5payloadsFixed"), true);- assertTerms(- leafReader.terms("field5payloadsFixed"),- leafReader.terms("field6payloadsVariable"),- true);- assertTerms(- leafReader.terms("field6payloadsVariable"),- leafReader.terms("field7payloadsFixedOffsets"),- true);- assertTerms(- leafReader.terms("field7payloadsFixedOffsets"),- leafReader.terms("field8payloadsVariableOffsets"),- true);- }- ir.close();- }-- // following code is almost an exact dup of code from TestDuelingCodecs: sorry!-- public void assertTerms(Terms leftTerms, Terms rightTerms, boolean deep) throws Exception {- if (leftTerms == null || rightTerms == null) {- assertNull(leftTerms);- assertNull(rightTerms);- return;- }- assertTermsStatistics(leftTerms, rightTerms);-- // NOTE: we don't assert hasOffsets/hasPositions/hasPayloads because they are allowed to be- // different-- boolean bothHaveFreqs = leftTerms.hasFreqs() && rightTerms.hasFreqs();- boolean bothHavePositions = leftTerms.hasPositions() && rightTerms.hasPositions();- TermsEnum leftTermsEnum = leftTerms.iterator();- TermsEnum rightTermsEnum = rightTerms.iterator();- assertTermsEnum(leftTermsEnum, rightTermsEnum, true, bothHaveFreqs, bothHavePositions);-- assertTermsSeeking(leftTerms, rightTerms);-- if (deep) {- int numIntersections = atLeast(3);- for (int i = 0; i < numIntersections; i++) {- String re = AutomatonTestUtil.randomRegexp(random());- Automaton a = new RegExp(re, RegExp.NONE).toAutomaton();- a = Operations.determinize(a, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);- CompiledAutomaton automaton = new CompiledAutomaton(a);- if (automaton.type == CompiledAutomaton.AUTOMATON_TYPE.NORMAL) {- // TODO: test start term too- TermsEnum leftIntersection = leftTerms.intersect(automaton, null);- TermsEnum rightIntersection = rightTerms.intersect(automaton, null);- assertTermsEnum(- leftIntersection, rightIntersection, rarely(), bothHaveFreqs, bothHavePositions);- }- }- }- }-- private void assertTermsSeeking(Terms leftTerms, Terms rightTerms) throws Exception {- TermsEnum leftEnum = null;- TermsEnum rightEnum = null;-- // just an upper bound- int numTests = atLeast(20);- Random random = random();-- // collect this number of terms from the left side- HashSet<BytesRef> tests = new HashSet<>();- int numPasses = 0;- while (numPasses < 10 && tests.size() < numTests) {- leftEnum = leftTerms.iterator();- BytesRef term = null;- while ((term = leftEnum.next()) != null) {- int code = random.nextInt(10);- if (code == 0) {- // the term- tests.add(BytesRef.deepCopyOf(term));- } else if (code == 1) {- // truncated subsequence of term- term = BytesRef.deepCopyOf(term);- if (term.length > 0) {- // truncate it- term.length = random.nextInt(term.length);- }- } else if (code == 2) {- // term, but ensure a non-zero offset- byte[] newbytes = new byte[term.length + 5];- System.arraycopy(term.bytes, term.offset, newbytes, 5, term.length);- tests.add(new BytesRef(newbytes, 5, term.length));- }- }- numPasses++;- }-- ArrayList<BytesRef> shuffledTests = new ArrayList<>(tests);- Collections.shuffle(shuffledTests, random);-- for (BytesRef b : shuffledTests) {- leftEnum = leftTerms.iterator();- rightEnum = rightTerms.iterator();-- assertEquals(leftEnum.seekExact(b), rightEnum.seekExact(b));- assertEquals(leftEnum.seekExact(b), rightEnum.seekExact(b));-- SeekStatus leftStatus;- SeekStatus rightStatus;-- leftStatus = leftEnum.seekCeil(b);- rightStatus = rightEnum.seekCeil(b);- assertEquals(leftStatus, rightStatus);- if (leftStatus != SeekStatus.END) {- assertEquals(leftEnum.term(), rightEnum.term());- }-- leftStatus = leftEnum.seekCeil(b);- rightStatus = rightEnum.seekCeil(b);- assertEquals(leftStatus, rightStatus);- if (leftStatus != SeekStatus.END) {- assertEquals(leftEnum.term(), rightEnum.term());- }- }- }-- /** checks collection-level statistics on Terms */- public void assertTermsStatistics(Terms leftTerms, Terms rightTerms) throws Exception {- assertEquals(leftTerms.getDocCount(), rightTerms.getDocCount());- assertEquals(leftTerms.getSumDocFreq(), rightTerms.getSumDocFreq());- if (leftTerms.hasFreqs() && rightTerms.hasFreqs()) {- assertEquals(leftTerms.getSumTotalTermFreq(), rightTerms.getSumTotalTermFreq());- }- if (leftTerms.size() != -1 && rightTerms.size() != -1) {- assertEquals(leftTerms.size(), rightTerms.size());- }- }-- /**- * checks the terms enum sequentially if deep is false, it does a 'shallow' test that doesnt go- * down to the docsenums- */- public void assertTermsEnum(- TermsEnum leftTermsEnum,- TermsEnum rightTermsEnum,- boolean deep,- boolean hasFreqs,- boolean hasPositions)- throws Exception {- BytesRef term;- PostingsEnum leftPositions = null;- PostingsEnum rightPositions = null;- PostingsEnum leftDocs = null;- PostingsEnum rightDocs = null;-- while ((term = leftTermsEnum.next()) != null) {- assertEquals(term, rightTermsEnum.next());- assertTermStats(leftTermsEnum, rightTermsEnum, hasFreqs);- if (deep) {- if (hasPositions) {- // with payloads + off- assertDocsAndPositionsEnum(- leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.ALL),- rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.ALL));-- assertPositionsSkipping(- leftTermsEnum.docFreq(),- leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.ALL),- rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.ALL));- // with payloads only- assertDocsAndPositionsEnum(- leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.PAYLOADS),- rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.PAYLOADS));-- assertPositionsSkipping(- leftTermsEnum.docFreq(),- leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.PAYLOADS),- rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.PAYLOADS));-- // with offsets only- assertDocsAndPositionsEnum(- leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.OFFSETS),- rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.OFFSETS));-- assertPositionsSkipping(- leftTermsEnum.docFreq(),- leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.OFFSETS),- rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.OFFSETS));-- // with positions only- assertDocsAndPositionsEnum(- leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.POSITIONS),- rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.POSITIONS));-- assertPositionsSkipping(- leftTermsEnum.docFreq(),- leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.POSITIONS),- rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.POSITIONS));- }-- // with freqs:- assertDocsEnum(- leftDocs = leftTermsEnum.postings(leftDocs),- rightDocs = rightTermsEnum.postings(rightDocs));-- // w/o freqs:- assertDocsEnum(- leftDocs = leftTermsEnum.postings(leftDocs, PostingsEnum.NONE),- rightDocs = rightTermsEnum.postings(rightDocs, PostingsEnum.NONE));-- // with freqs:- assertDocsSkipping(- leftTermsEnum.docFreq(),- leftDocs = leftTermsEnum.postings(leftDocs),- rightDocs = rightTermsEnum.postings(rightDocs));-- // w/o freqs:- assertDocsSkipping(- leftTermsEnum.docFreq(),- leftDocs = leftTermsEnum.postings(leftDocs, PostingsEnum.NONE),- rightDocs = rightTermsEnum.postings(rightDocs, PostingsEnum.NONE));- }- }- assertNull(rightTermsEnum.next());- }-- /** checks term-level statistics */- public void assertTermStats(- TermsEnum leftTermsEnum, TermsEnum rightTermsEnum, boolean bothHaveFreqs) throws Exception {- assertEquals(leftTermsEnum.docFreq(), rightTermsEnum.docFreq());- if (bothHaveFreqs) {- assertEquals(leftTermsEnum.totalTermFreq(), rightTermsEnum.totalTermFreq());- }- }-- /** checks docs + freqs + positions + payloads, sequentially */- public void assertDocsAndPositionsEnum(PostingsEnum leftDocs, PostingsEnum rightDocs)- throws Exception {- assertNotNull(leftDocs);- assertNotNull(rightDocs);- assertEquals(-1, leftDocs.docID());- assertEquals(-1, rightDocs.docID());- int docid;- while ((docid = leftDocs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {- assertEquals(docid, rightDocs.nextDoc());- int freq = leftDocs.freq();- assertEquals(freq, rightDocs.freq());- for (int i = 0; i < freq; i++) {- assertEquals(leftDocs.nextPosition(), rightDocs.nextPosition());- // we don't assert offsets/payloads, they are allowed to be different- }- }- assertEquals(DocIdSetIterator.NO_MORE_DOCS, rightDocs.nextDoc());- }-- /** checks docs + freqs, sequentially */- public void assertDocsEnum(PostingsEnum leftDocs, PostingsEnum rightDocs) throws Exception {- if (leftDocs == null) {- assertNull(rightDocs);- return;- }- assertEquals(-1, leftDocs.docID());- assertEquals(-1, rightDocs.docID());- int docid;- while ((docid = leftDocs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {- assertEquals(docid, rightDocs.nextDoc());- // we don't assert freqs, they are allowed to be different- }- assertEquals(DocIdSetIterator.NO_MORE_DOCS, rightDocs.nextDoc());- }-- /** checks advancing docs */- public void assertDocsSkipping(int docFreq, PostingsEnum leftDocs, PostingsEnum rightDocs)- throws Exception {- if (leftDocs == null) {- assertNull(rightDocs);- return;- }- int docid = -1;- int averageGap = MAXDOC / (1 + docFreq);- int skipInterval = 16;-- while (true) {- if (random().nextBoolean()) {- // nextDoc()- docid = leftDocs.nextDoc();- assertEquals(docid, rightDocs.nextDoc());- } else {- // advance()- int skip =- docid + (int) Math.ceil(Math.abs(skipInterval + random().nextGaussian() * averageGap));- docid = leftDocs.advance(skip);- assertEquals(docid, rightDocs.advance(skip));- }-- if (docid == DocIdSetIterator.NO_MORE_DOCS) {- return;- }- // we don't assert freqs, they are allowed to be different- }- }-- /** checks advancing docs + positions */- public void assertPositionsSkipping(int docFreq, PostingsEnum leftDocs, PostingsEnum rightDocs)- throws Exception {- if (leftDocs == null || rightDocs == null) {- assertNull(leftDocs);- assertNull(rightDocs);- return;- }-- int docid = -1;- int averageGap = MAXDOC / (1 + docFreq);- int skipInterval = 16;-- while (true) {- if (random().nextBoolean()) {- // nextDoc()- docid = leftDocs.nextDoc();- assertEquals(docid, rightDocs.nextDoc());- } else {- // advance()- int skip =- docid + (int) Math.ceil(Math.abs(skipInterval + random().nextGaussian() * averageGap));- docid = leftDocs.advance(skip);- assertEquals(docid, rightDocs.advance(skip));- }-- if (docid == DocIdSetIterator.NO_MORE_DOCS) {- return;- }- int freq = leftDocs.freq();- assertEquals(freq, rightDocs.freq());- for (int i = 0; i < freq; i++) {- assertEquals(leftDocs.nextPosition(), rightDocs.nextPosition());- // we don't compare the payloads, it's allowed that one is empty etc- }- }- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestForUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestForUtil.javadeleted file mode 100644index 6c9bfebc20b..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestForUtil.java+++ /dev/null@@ -1,98 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import static org.apache.lucene.backward_codecs.lucene50.ForUtil.MAX_DATA_SIZE;-import static org.apache.lucene.backward_codecs.lucene50.ForUtil.MAX_ENCODED_SIZE;-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.BLOCK_SIZE;--import com.carrotsearch.randomizedtesting.generators.RandomNumbers;-import java.io.IOException;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.store.ByteBuffersDirectory;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.store.IndexOutput;-import org.apache.lucene.tests.util.LuceneTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.packed.PackedInts;--@Nightly // N-2 formats are only tested on nightly runs-public class TestForUtil extends LuceneTestCase {-- public void testEncodeDecode() throws IOException {- final int iterations = RandomNumbers.randomIntBetween(random(), 1, 1000);- final float acceptableOverheadRatio = random().nextFloat();- final int[] values = new int[iterations * BLOCK_SIZE];- for (int i = 0; i < iterations; ++i) {- final int bpv = random().nextInt(32);- if (bpv == 0) {- final int value = RandomNumbers.randomIntBetween(random(), 0, Integer.MAX_VALUE);- for (int j = 0; j < BLOCK_SIZE; ++j) {- values[i * BLOCK_SIZE + j] = value;- }- } else {- for (int j = 0; j < BLOCK_SIZE; ++j) {- values[i * BLOCK_SIZE + j] =- RandomNumbers.randomIntBetween(random(), 0, (int) PackedInts.maxValue(bpv));- }- }- }-- final Directory d = new ByteBuffersDirectory();- final long endPointer;-- {- // encode- IndexOutput out = EndiannessReverserUtil.createOutput(d, "test.bin", IOContext.DEFAULT);- final ForUtil forUtil = new ForUtil(acceptableOverheadRatio, out);-- for (int i = 0; i < iterations; ++i) {- // Although values after BLOCK_SIZE are garbage, we need to allocate extra bytes to avoid- // AIOOBE.- int[] block =- ArrayUtil.grow(ArrayUtil.copyOfSubArray(values, i * BLOCK_SIZE, (i + 1) * BLOCK_SIZE));- forUtil.writeBlock(ArrayUtil.grow(block, MAX_DATA_SIZE), new byte[MAX_ENCODED_SIZE], out);- }- endPointer = out.getFilePointer();- out.close();- }-- {- // decode- IndexInput in = EndiannessReverserUtil.openInput(d, "test.bin", IOContext.READONCE);- final ForUtil forUtil = new ForUtil(in);- for (int i = 0; i < iterations; ++i) {- if (random().nextBoolean()) {- forUtil.skipBlock(in);- continue;- }- final int[] restored = new int[MAX_DATA_SIZE];- forUtil.readBlock(in, new byte[MAX_ENCODED_SIZE], restored);- assertArrayEquals(- ArrayUtil.copyOfSubArray(values, i * BLOCK_SIZE, (i + 1) * BLOCK_SIZE),- ArrayUtil.copyOfSubArray(restored, 0, BLOCK_SIZE));- }- assertEquals(endPointer, in.getFilePointer());- in.close();- }-- d.close();- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50CompoundFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50CompoundFormat.javadeleted file mode 100644index 69c38210c7c..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50CompoundFormat.java+++ /dev/null@@ -1,31 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import org.apache.lucene.backward_codecs.lucene87.Lucene87RWCodec;-import org.apache.lucene.codecs.Codec;-import org.apache.lucene.tests.index.BaseCompoundFormatTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;--@Nightly // N-2 formats are only tested on nightly runs-public class TestLucene50CompoundFormat extends BaseCompoundFormatTestCase {-- @Override- protected Codec getCodec() {- return new Lucene87RWCodec();- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50LiveDocsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50LiveDocsFormat.javadeleted file mode 100644index fca261f2a8a..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50LiveDocsFormat.java+++ /dev/null@@ -1,31 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import org.apache.lucene.backward_codecs.lucene86.Lucene86RWCodec;-import org.apache.lucene.codecs.Codec;-import org.apache.lucene.tests.index.BaseLiveDocsFormatTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;--@Nightly // N-2 formats are only tested on nightly runs-public class TestLucene50LiveDocsFormat extends BaseLiveDocsFormatTestCase {-- @Override- protected Codec getCodec() {- return new Lucene86RWCodec();- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50StoredFieldsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50StoredFieldsFormat.javadeleted file mode 100644index 51fd2a6c50d..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50StoredFieldsFormat.java+++ /dev/null@@ -1,30 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import org.apache.lucene.backward_codecs.lucene86.Lucene86RWCodec;-import org.apache.lucene.codecs.Codec;-import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;--@Nightly // N-2 formats are only tested on nightly runs-public class TestLucene50StoredFieldsFormat extends BaseStoredFieldsFormatTestCase {- @Override- protected Codec getCodec() {- return new Lucene86RWCodec();- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50StoredFieldsFormatHighCompression.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50StoredFieldsFormatHighCompression.javadeleted file mode 100644index 4089a28e977..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50StoredFieldsFormatHighCompression.java+++ /dev/null@@ -1,86 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import com.carrotsearch.randomizedtesting.generators.RandomPicks;-import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat.Mode;-import org.apache.lucene.backward_codecs.lucene86.Lucene86RWCodec;-import org.apache.lucene.codecs.Codec;-import org.apache.lucene.document.Document;-import org.apache.lucene.document.StoredField;-import org.apache.lucene.index.DirectoryReader;-import org.apache.lucene.index.IndexWriter;-import org.apache.lucene.index.IndexWriterConfig;-import org.apache.lucene.index.StoredFields;-import org.apache.lucene.store.Directory;-import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;--@Nightly // N-2 formats are only tested on nightly runs-public class TestLucene50StoredFieldsFormatHighCompression extends BaseStoredFieldsFormatTestCase {- @Override- protected Codec getCodec() {- return new Lucene86RWCodec(Mode.BEST_COMPRESSION);- }-- /**- * Change compression params (leaving it the same for old segments) and tests that nothing breaks.- */- public void testMixedCompressions() throws Exception {- Directory dir = newDirectory();- for (int i = 0; i < 10; i++) {- IndexWriterConfig iwc = newIndexWriterConfig();- iwc.setCodec(new Lucene86RWCodec(RandomPicks.randomFrom(random(), Mode.values())));- IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig());- Document doc = new Document();- doc.add(new StoredField("field1", "value1"));- doc.add(new StoredField("field2", "value2"));- iw.addDocument(doc);- if (random().nextInt(4) == 0) {- iw.forceMerge(1);- }- iw.commit();- iw.close();- }-- DirectoryReader ir = DirectoryReader.open(dir);- assertEquals(10, ir.numDocs());- StoredFields storedFields = ir.storedFields();- for (int i = 0; i < 10; i++) {- Document doc = storedFields.document(i);- assertEquals("value1", doc.get("field1"));- assertEquals("value2", doc.get("field2"));- }- ir.close();- // checkindex- dir.close();- }-- public void testInvalidOptions() {- expectThrows(- NullPointerException.class,- () -> {- new Lucene86RWCodec(null);- });-- expectThrows(- NullPointerException.class,- () -> {- new Lucene50StoredFieldsFormat(null);- });- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50StoredFieldsFormatMergeInstance.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50StoredFieldsFormatMergeInstance.javadeleted file mode 100644index 5034e4032d6..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50StoredFieldsFormatMergeInstance.java+++ /dev/null@@ -1,29 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import org.apache.lucene.tests.util.LuceneTestCase.Nightly;--/** Test the merge instance of the Lucene50 stored fields format. */-@Nightly // N-2 formats are only tested on nightly runs-public class TestLucene50StoredFieldsFormatMergeInstance extends TestLucene50StoredFieldsFormat {-- @Override- protected boolean shouldTestMergeInstance() {- return true;- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50TermVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50TermVectorsFormat.javadeleted file mode 100644index 388707407c8..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50TermVectorsFormat.java+++ /dev/null@@ -1,31 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50;--import org.apache.lucene.backward_codecs.lucene87.Lucene87RWCodec;-import org.apache.lucene.codecs.Codec;-import org.apache.lucene.tests.index.BaseTermVectorsFormatTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;--@Nightly // N-2 formats are only tested on nightly runs-public class TestLucene50TermVectorsFormat extends BaseTermVectorsFormatTestCase {-- @Override- protected Codec getCodec() {- return new Lucene87RWCodec();- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndexWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndexWriter.javadeleted file mode 100644index 857505fd1f2..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndexWriter.java+++ /dev/null@@ -1,210 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50.compressing;--import static org.apache.lucene.backward_codecs.lucene50.compressing.FieldsIndexReader.VERSION_CURRENT;--import java.io.Closeable;-import java.io.IOException;-import java.util.ArrayList;-import java.util.List;-import org.apache.lucene.backward_codecs.packed.LegacyDirectMonotonicReader;-import org.apache.lucene.backward_codecs.packed.LegacyDirectMonotonicWriter;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.Codec;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.index.CorruptIndexException;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.store.ChecksumIndexInput;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IndexOutput;-import org.apache.lucene.util.IOUtils;--/**- * Efficient index format for block-based {@link Codec}s.- *- * <p>For each block of compressed stored fields, this stores the first document of the block and- * the start pointer of the block in a {@link LegacyDirectMonotonicWriter}. At read time, the docID- * is binary-searched in the {@link LegacyDirectMonotonicWriter} that records doc IDS, and the- * returned index is used to look up the start pointer in the {@link LegacyDirectMonotonicReader}- * that records start pointers.- *- * @lucene.internal- */-public final class FieldsIndexWriter implements Closeable {-- private final Directory dir;- private final String name;- private final String suffix;- private final String extension;- private final String codecName;- private final byte[] id;- private final int blockShift;- private final IOContext ioContext;- private IndexOutput docsOut;- private IndexOutput filePointersOut;- private int totalDocs;- private int totalChunks;- private long previousFP;-- FieldsIndexWriter(- Directory dir,- String name,- String suffix,- String extension,- String codecName,- byte[] id,- int blockShift,- IOContext ioContext)- throws IOException {- this.dir = dir;- this.name = name;- this.suffix = suffix;- this.extension = extension;- this.codecName = codecName;- this.id = id;- this.blockShift = blockShift;- this.ioContext = ioContext;- this.docsOut =- EndiannessReverserUtil.createTempOutput(dir, name, codecName + "-doc_ids", ioContext);- boolean success = false;- try {- CodecUtil.writeHeader(docsOut, codecName + "Docs", VERSION_CURRENT);- filePointersOut =- EndiannessReverserUtil.createTempOutput(- dir, name, codecName + "file_pointers", ioContext);- CodecUtil.writeHeader(filePointersOut, codecName + "FilePointers", VERSION_CURRENT);- success = true;- } finally {- if (success == false) {- close();- }- }- }-- void writeIndex(int numDocs, long startPointer) throws IOException {- assert startPointer >= previousFP;- docsOut.writeVInt(numDocs);- filePointersOut.writeVLong(startPointer - previousFP);- previousFP = startPointer;- totalDocs += numDocs;- totalChunks++;- }-- void finish(int numDocs, long maxPointer, IndexOutput metaOut) throws IOException {- if (numDocs != totalDocs) {- throw new IllegalStateException("Expected " + numDocs + " docs, but got " + totalDocs);- }- CodecUtil.writeFooter(docsOut);- CodecUtil.writeFooter(filePointersOut);- IOUtils.close(docsOut, filePointersOut);-- try (IndexOutput dataOut =- EndiannessReverserUtil.createOutput(- dir, IndexFileNames.segmentFileName(name, suffix, extension), ioContext)) {- CodecUtil.writeIndexHeader(dataOut, codecName + "Idx", VERSION_CURRENT, id, suffix);-- metaOut.writeInt(numDocs);- metaOut.writeInt(blockShift);- metaOut.writeInt(totalChunks + 1);- metaOut.writeLong(dataOut.getFilePointer());-- try (ChecksumIndexInput docsIn =- EndiannessReverserUtil.openChecksumInput(dir, docsOut.getName(), IOContext.READONCE)) {- CodecUtil.checkHeader(docsIn, codecName + "Docs", VERSION_CURRENT, VERSION_CURRENT);- Throwable priorE = null;- try {- final LegacyDirectMonotonicWriter docs =- LegacyDirectMonotonicWriter.getInstance(- metaOut, dataOut, totalChunks + 1, blockShift);- long doc = 0;- docs.add(doc);- for (int i = 0; i < totalChunks; ++i) {- doc += docsIn.readVInt();- docs.add(doc);- }- docs.finish();- if (doc != totalDocs) {- throw new CorruptIndexException("Docs don't add up", docsIn);- }- } catch (Throwable e) {- priorE = e;- } finally {- CodecUtil.checkFooter(docsIn, priorE);- }- }- dir.deleteFile(docsOut.getName());- docsOut = null;-- metaOut.writeLong(dataOut.getFilePointer());- try (ChecksumIndexInput filePointersIn =- EndiannessReverserUtil.openChecksumInput(- dir, filePointersOut.getName(), IOContext.READONCE)) {- CodecUtil.checkHeader(- filePointersIn, codecName + "FilePointers", VERSION_CURRENT, VERSION_CURRENT);- Throwable priorE = null;- try {- final LegacyDirectMonotonicWriter filePointers =- LegacyDirectMonotonicWriter.getInstance(- metaOut, dataOut, totalChunks + 1, blockShift);- long fp = 0;- for (int i = 0; i < totalChunks; ++i) {- fp += filePointersIn.readVLong();- filePointers.add(fp);- }- if (maxPointer < fp) {- throw new CorruptIndexException("File pointers don't add up", filePointersIn);- }- filePointers.add(maxPointer);- filePointers.finish();- } catch (Throwable e) {- priorE = e;- } finally {- CodecUtil.checkFooter(filePointersIn, priorE);- }- }- dir.deleteFile(filePointersOut.getName());- filePointersOut = null;-- metaOut.writeLong(dataOut.getFilePointer());- metaOut.writeLong(maxPointer);-- CodecUtil.writeFooter(dataOut);- }- }-- @Override- public void close() throws IOException {- try {- IOUtils.close(docsOut, filePointersOut);- } finally {- List<String> fileNames = new ArrayList<>();- if (docsOut != null) {- fileNames.add(docsOut.getName());- }- if (filePointersOut != null) {- fileNames.add(filePointersOut.getName());- }- try {- IOUtils.deleteFiles(dir, fileNames);- } finally {- docsOut = filePointersOut = null;- }- }- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingStoredFieldsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingStoredFieldsWriter.javadeleted file mode 100644index 0c2c7c2b6fa..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingStoredFieldsWriter.java+++ /dev/null@@ -1,507 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50.compressing;--import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.BYTE_ARR;-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.FIELDS_EXTENSION;-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.INDEX_CODEC_NAME;-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.INDEX_EXTENSION;-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.META_EXTENSION;-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.NUMERIC_DOUBLE;-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.NUMERIC_FLOAT;-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.NUMERIC_INT;-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.NUMERIC_LONG;-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.STRING;-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.TYPE_BITS;-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.VERSION_CURRENT;--import java.io.IOException;-import org.apache.lucene.backward_codecs.compressing.CompressionMode;-import org.apache.lucene.backward_codecs.compressing.Compressor;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.StoredFieldsWriter;-import org.apache.lucene.index.FieldInfo;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.SegmentInfo;-import org.apache.lucene.store.ByteBuffersDataOutput;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IndexOutput;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.BitUtil;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.IOUtils;-import org.apache.lucene.util.packed.PackedInts;--/**- * {@link StoredFieldsWriter} impl for {@link Lucene50CompressingStoredFieldsFormat}.- *- * @lucene.experimental- */-public final class Lucene50CompressingStoredFieldsWriter extends StoredFieldsWriter {-- private final String segment;- private FieldsIndexWriter indexWriter;- private IndexOutput metaStream, fieldsStream;-- private Compressor compressor;- private final int chunkSize;- private final int maxDocsPerChunk;-- private final ByteBuffersDataOutput bufferedDocs;- private int[] numStoredFields; // number of stored fields- private int[] endOffsets; // end offsets in bufferedDocs- private int docBase; // doc ID at the beginning of the chunk- private int numBufferedDocs; // docBase + numBufferedDocs == current doc ID-- private long numChunks; // number of written chunks- private long numDirtyChunks; // number of incomplete compressed blocks written- private long numDirtyDocs; // cumulative number of missing docs in incomplete chunks-- /** Sole constructor. */- Lucene50CompressingStoredFieldsWriter(- Directory directory,- SegmentInfo si,- String segmentSuffix,- IOContext context,- String formatName,- CompressionMode compressionMode,- int chunkSize,- int maxDocsPerChunk,- int blockShift)- throws IOException {- assert directory != null;- this.segment = si.name;- this.compressor = compressionMode.newCompressor();- this.chunkSize = chunkSize;- this.maxDocsPerChunk = maxDocsPerChunk;- this.docBase = 0;- this.bufferedDocs = ByteBuffersDataOutput.newResettableInstance();- this.numStoredFields = new int[16];- this.endOffsets = new int[16];- this.numBufferedDocs = 0;-- boolean success = false;- try {- metaStream =- EndiannessReverserUtil.createOutput(- directory,- IndexFileNames.segmentFileName(segment, segmentSuffix, META_EXTENSION),- context);- CodecUtil.writeIndexHeader(- metaStream, INDEX_CODEC_NAME + "Meta", VERSION_CURRENT, si.getId(), segmentSuffix);- assert CodecUtil.indexHeaderLength(INDEX_CODEC_NAME + "Meta", segmentSuffix)- == metaStream.getFilePointer();-- fieldsStream =- EndiannessReverserUtil.createOutput(- directory,- IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION),- context);- CodecUtil.writeIndexHeader(- fieldsStream, formatName, VERSION_CURRENT, si.getId(), segmentSuffix);- assert CodecUtil.indexHeaderLength(formatName, segmentSuffix)- == fieldsStream.getFilePointer();-- indexWriter =- new FieldsIndexWriter(- directory,- segment,- segmentSuffix,- INDEX_EXTENSION,- INDEX_CODEC_NAME,- si.getId(),- blockShift,- context);-- metaStream.writeVInt(chunkSize);- metaStream.writeVInt(PackedInts.VERSION_CURRENT);-- success = true;- } finally {- if (!success) {- IOUtils.closeWhileHandlingException(metaStream, fieldsStream, indexWriter);- }- }- }-- @Override- public void close() throws IOException {- try {- IOUtils.close(metaStream, fieldsStream, indexWriter, compressor);- } finally {- metaStream = null;- fieldsStream = null;- indexWriter = null;- compressor = null;- }- }-- private int numStoredFieldsInDoc;-- @Override- public void startDocument() throws IOException {}-- @Override- public void finishDocument() throws IOException {- if (numBufferedDocs == this.numStoredFields.length) {- final int newLength = ArrayUtil.oversize(numBufferedDocs + 1, 4);- this.numStoredFields = ArrayUtil.growExact(this.numStoredFields, newLength);- endOffsets = ArrayUtil.growExact(endOffsets, newLength);- }- this.numStoredFields[numBufferedDocs] = numStoredFieldsInDoc;- numStoredFieldsInDoc = 0;- endOffsets[numBufferedDocs] = Math.toIntExact(bufferedDocs.size());- ++numBufferedDocs;- if (triggerFlush()) {- flush(false);- }- }-- private static void saveInts(int[] values, int length, DataOutput out) throws IOException {- assert length > 0;- if (length == 1) {- out.writeVInt(values[0]);- } else {- boolean allEqual = true;- for (int i = 1; i < length; ++i) {- if (values[i] != values[0]) {- allEqual = false;- break;- }- }- if (allEqual) {- out.writeVInt(0);- out.writeVInt(values[0]);- } else {- long max = 0;- for (int i = 0; i < length; ++i) {- max |= values[i];- }- final int bitsRequired = PackedInts.bitsRequired(max);- out.writeVInt(bitsRequired);- final PackedInts.Writer w =- PackedInts.getWriterNoHeader(out, PackedInts.Format.PACKED, length, bitsRequired, 1);- for (int i = 0; i < length; ++i) {- w.add(values[i]);- }- w.finish();- }- }- }-- private void writeHeader(- int docBase,- int numBufferedDocs,- int[] numStoredFields,- int[] lengths,- boolean sliced,- boolean dirtyChunk)- throws IOException {- final int slicedBit = sliced ? 1 : 0;- final int dirtyBit = dirtyChunk ? 2 : 0;-- // save docBase and numBufferedDocs- fieldsStream.writeVInt(docBase);- fieldsStream.writeVInt((numBufferedDocs << 2) | dirtyBit | slicedBit);-- // save numStoredFields- saveInts(numStoredFields, numBufferedDocs, fieldsStream);-- // save lengths- saveInts(lengths, numBufferedDocs, fieldsStream);- }-- private boolean triggerFlush() {- return bufferedDocs.size() >= chunkSize- || // chunks of at least chunkSize bytes- numBufferedDocs >= maxDocsPerChunk;- }-- private void flush(boolean force) throws IOException {- assert triggerFlush() != force;- numChunks++;- if (force) {- numDirtyChunks++; // incomplete: we had to force this flush- numDirtyDocs += numBufferedDocs;- }- indexWriter.writeIndex(numBufferedDocs, fieldsStream.getFilePointer());-- // transform end offsets into lengths- final int[] lengths = endOffsets;- for (int i = numBufferedDocs - 1; i > 0; --i) {- lengths[i] = endOffsets[i] - endOffsets[i - 1];- assert lengths[i] >= 0;- }- final boolean sliced = bufferedDocs.size() >= 2 * chunkSize;- writeHeader(docBase, numBufferedDocs, numStoredFields, lengths, sliced, force);-- // compress stored fields to fieldsStream.- //- // TODO: do we need to slice it since we already have the slices in the buffer? Perhaps- // we should use max-block-bits restriction on the buffer itself, then we won't have to check it- // here.- byte[] content = bufferedDocs.toArrayCopy();- bufferedDocs.reset();-- if (sliced) {- // big chunk, slice it- for (int compressed = 0; compressed < content.length; compressed += chunkSize) {- compressor.compress(- content, compressed, Math.min(chunkSize, content.length - compressed), fieldsStream);- }- } else {- compressor.compress(content, 0, content.length, fieldsStream);- }-- // reset- docBase += numBufferedDocs;- numBufferedDocs = 0;- bufferedDocs.reset();- }-- @Override- public void writeField(FieldInfo info, int value) throws IOException {- ++numStoredFieldsInDoc;- final long infoAndBits = (((long) info.number) << TYPE_BITS) | NUMERIC_INT;- bufferedDocs.writeVLong(infoAndBits);- bufferedDocs.writeZInt(value);- }-- @Override- public void writeField(FieldInfo info, long value) throws IOException {- ++numStoredFieldsInDoc;- final long infoAndBits = (((long) info.number) << TYPE_BITS) | NUMERIC_LONG;- bufferedDocs.writeVLong(infoAndBits);- writeTLong(EndiannessReverserUtil.wrapDataOutput(bufferedDocs), value);- }-- @Override- public void writeField(FieldInfo info, float value) throws IOException {- ++numStoredFieldsInDoc;- final long infoAndBits = (((long) info.number) << TYPE_BITS) | NUMERIC_FLOAT;- bufferedDocs.writeVLong(infoAndBits);- writeZFloat(EndiannessReverserUtil.wrapDataOutput(bufferedDocs), value);- }-- @Override- public void writeField(FieldInfo info, double value) throws IOException {- ++numStoredFieldsInDoc;- final long infoAndBits = (((long) info.number) << TYPE_BITS) | NUMERIC_DOUBLE;- bufferedDocs.writeVLong(infoAndBits);- writeZDouble(EndiannessReverserUtil.wrapDataOutput(bufferedDocs), value);- }-- @Override- public void writeField(FieldInfo info, BytesRef value) throws IOException {- ++numStoredFieldsInDoc;- final long infoAndBits = (((long) info.number) << TYPE_BITS) | BYTE_ARR;- bufferedDocs.writeVLong(infoAndBits);- bufferedDocs.writeVInt(value.length);- bufferedDocs.writeBytes(value.bytes, value.offset, value.length);- }-- @Override- public void writeField(FieldInfo info, String value) throws IOException {- ++numStoredFieldsInDoc;- final long infoAndBits = (((long) info.number) << TYPE_BITS) | STRING;- bufferedDocs.writeVLong(infoAndBits);- bufferedDocs.writeString(value);- }-- // -0 isn't compressed.- static final int NEGATIVE_ZERO_FLOAT = Float.floatToIntBits(-0f);- static final long NEGATIVE_ZERO_DOUBLE = Double.doubleToLongBits(-0d);-- // for compression of timestamps- static final long SECOND = 1000L;- static final long HOUR = 60 * 60 * SECOND;- static final long DAY = 24 * HOUR;- static final int SECOND_ENCODING = 0x40;- static final int HOUR_ENCODING = 0x80;- static final int DAY_ENCODING = 0xC0;-- /**- * Writes a float in a variable-length format. Writes between one and five bytes. Small integral- * values typically take fewer bytes.- *- * <p>ZFloat --> Header, Bytes*?- *- * <ul>- * <li>Header --> {@link DataOutput#writeByte Uint8}. When it is equal to 0xFF then the value- * is negative and stored in the next 4 bytes. Otherwise if the first bit is set then the- * other bits in the header encode the value plus one and no other bytes are read.- * Otherwise, the value is a positive float value whose first byte is the header, and 3- * bytes need to be read to complete it.- * <li>Bytes --> Potential additional bytes to read depending on the header.- * </ul>- */- static void writeZFloat(DataOutput out, float f) throws IOException {- int intVal = (int) f;- final int floatBits = Float.floatToIntBits(f);-- if (f == intVal && intVal >= -1 && intVal <= 0x7D && floatBits != NEGATIVE_ZERO_FLOAT) {- // small integer value [-1..125]: single byte- out.writeByte((byte) (0x80 | (1 + intVal)));- } else if ((floatBits >>> 31) == 0) {- // other positive floats: 4 bytes- out.writeInt(floatBits);- } else {- // other negative float: 5 bytes- out.writeByte((byte) 0xFF);- out.writeInt(floatBits);- }- }-- /**- * Writes a float in a variable-length format. Writes between one and five bytes. Small integral- * values typically take fewer bytes.- *- * <p>ZFloat --> Header, Bytes*?- *- * <ul>- * <li>Header --> {@link DataOutput#writeByte Uint8}. When it is equal to 0xFF then the value- * is negative and stored in the next 8 bytes. When it is equal to 0xFE then the value is- * stored as a float in the next 4 bytes. Otherwise if the first bit is set then the other- * bits in the header encode the value plus one and no other bytes are read. Otherwise, the- * value is a positive float value whose first byte is the header, and 7 bytes need to be- * read to complete it.- * <li>Bytes --> Potential additional bytes to read depending on the header.- * </ul>- */- static void writeZDouble(DataOutput out, double d) throws IOException {- int intVal = (int) d;- final long doubleBits = Double.doubleToLongBits(d);-- if (d == intVal && intVal >= -1 && intVal <= 0x7C && doubleBits != NEGATIVE_ZERO_DOUBLE) {- // small integer value [-1..124]: single byte- out.writeByte((byte) (0x80 | (intVal + 1)));- return;- } else if (d == (float) d) {- // d has an accurate float representation: 5 bytes- out.writeByte((byte) 0xFE);- out.writeInt(Float.floatToIntBits((float) d));- } else if ((doubleBits >>> 63) == 0) {- // other positive doubles: 8 bytes- out.writeLong(doubleBits);- } else {- // other negative doubles: 9 bytes- out.writeByte((byte) 0xFF);- out.writeLong(doubleBits);- }- }-- /**- * Writes a long in a variable-length format. Writes between one and ten bytes. Small values or- * values representing timestamps with day, hour or second precision typically require fewer- * bytes.- *- * <p>ZLong --> Header, Bytes*?- *- * <ul>- * <li>Header --> The first two bits indicate the compression scheme:- * <ul>- * <li>00 - uncompressed- * <li>01 - multiple of 1000 (second)- * <li>10 - multiple of 3600000 (hour)- * <li>11 - multiple of 86400000 (day)- * </ul>- * Then the next bit is a continuation bit, indicating whether more bytes need to be read,- * and the last 5 bits are the lower bits of the encoded value. In order to reconstruct the- * value, you need to combine the 5 lower bits of the header with a vLong in the next bytes- * (if the continuation bit is set to 1). Then {@link BitUtil#zigZagDecode(int)- * zigzag-decode} it and finally multiply by the multiple corresponding to the compression- * scheme.- * <li>Bytes --> Potential additional bytes to read depending on the header.- * </ul>- */- // T for "timestamp"- static void writeTLong(DataOutput out, long l) throws IOException {- int header;- if (l % SECOND != 0) {- header = 0;- } else if (l % DAY == 0) {- // timestamp with day precision- header = DAY_ENCODING;- l /= DAY;- } else if (l % HOUR == 0) {- // timestamp with hour precision, or day precision with a timezone- header = HOUR_ENCODING;- l /= HOUR;- } else {- // timestamp with second precision- header = SECOND_ENCODING;- l /= SECOND;- }-- final long zigZagL = BitUtil.zigZagEncode(l);- header |= (zigZagL & 0x1F); // last 5 bits- final long upperBits = zigZagL >>> 5;- if (upperBits != 0) {- header |= 0x20;- }- out.writeByte((byte) header);- if (upperBits != 0) {- out.writeVLong(upperBits);- }- }-- @Override- public void finish(int numDocs) throws IOException {- if (numBufferedDocs > 0) {- flush(true);- } else {- assert bufferedDocs.size() == 0;- }- if (docBase != numDocs) {- throw new RuntimeException(- "Wrote " + docBase + " docs, finish called with numDocs=" + numDocs);- }- indexWriter.finish(numDocs, fieldsStream.getFilePointer(), metaStream);- metaStream.writeVLong(numChunks);- metaStream.writeVLong(numDirtyChunks);- metaStream.writeVLong(numDirtyDocs);- CodecUtil.writeFooter(metaStream);- CodecUtil.writeFooter(fieldsStream);- assert bufferedDocs.size() == 0;- }-- // bulk merge is scary: its caused corruption bugs in the past.- // we try to be extra safe with this impl, but add an escape hatch to- // have a workaround for undiscovered bugs.- static final String BULK_MERGE_ENABLED_SYSPROP =- Lucene50CompressingStoredFieldsWriter.class.getName() + ".enableBulkMerge";- static final boolean BULK_MERGE_ENABLED;-- static {- boolean v = true;- try {- v = Boolean.parseBoolean(System.getProperty(BULK_MERGE_ENABLED_SYSPROP, "true"));- } catch (SecurityException _) {- }- BULK_MERGE_ENABLED = v;- }-- @Override- public long ramBytesUsed() {- return bufferedDocs.ramBytesUsed()- + numStoredFields.length * Integer.BYTES- + endOffsets.length * Integer.BYTES;- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsWriter.javadeleted file mode 100644index 9a5e89695c3..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsWriter.java+++ /dev/null@@ -1,809 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50.compressing;--import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.FLAGS_BITS;-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.OFFSETS;-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.PACKED_BLOCK_SIZE;-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.PAYLOADS;-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.POSITIONS;-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.VECTORS_EXTENSION;-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.VECTORS_INDEX_CODEC_NAME;-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.VECTORS_INDEX_EXTENSION;-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.VECTORS_META_EXTENSION;-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.VERSION_CURRENT;--import java.io.IOException;-import java.util.ArrayDeque;-import java.util.Arrays;-import java.util.Collection;-import java.util.Deque;-import java.util.Iterator;-import java.util.List;-import java.util.SortedSet;-import java.util.TreeSet;-import org.apache.lucene.backward_codecs.compressing.CompressionMode;-import org.apache.lucene.backward_codecs.compressing.Compressor;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.TermVectorsWriter;-import org.apache.lucene.index.FieldInfo;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.SegmentInfo;-import org.apache.lucene.store.ByteBuffersDataOutput;-import org.apache.lucene.store.DataInput;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IndexOutput;-import org.apache.lucene.util.Accountable;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.IOUtils;-import org.apache.lucene.util.StringHelper;-import org.apache.lucene.util.packed.BlockPackedWriter;-import org.apache.lucene.util.packed.PackedInts;--/**- * {@link TermVectorsWriter} for {@link Lucene50CompressingTermVectorsFormat}.- *- * @lucene.experimental- */-public final class Lucene50CompressingTermVectorsWriter extends TermVectorsWriter {-- private final String segment;- private FieldsIndexWriter indexWriter;- private IndexOutput metaStream, vectorsStream;-- private final Compressor compressor;- private final int chunkSize;-- private long numChunks; // number of chunks- private long numDirtyChunks; // number of incomplete compressed blocks written- private long numDirtyDocs; // cumulative number of docs in incomplete chunks-- /** a pending doc */- private class DocData {- final int numFields;- final Deque<FieldData> fields;- final int posStart, offStart, payStart;-- DocData(int numFields, int posStart, int offStart, int payStart) {- this.numFields = numFields;- this.fields = new ArrayDeque<>(numFields);- this.posStart = posStart;- this.offStart = offStart;- this.payStart = payStart;- }-- FieldData addField(- int fieldNum, int numTerms, boolean positions, boolean offsets, boolean payloads) {- final FieldData field;- if (fields.isEmpty()) {- field =- new FieldData(- fieldNum, numTerms, positions, offsets, payloads, posStart, offStart, payStart);- } else {- final FieldData last = fields.getLast();- final int posStart = last.posStart + (last.hasPositions ? last.totalPositions : 0);- final int offStart = last.offStart + (last.hasOffsets ? last.totalPositions : 0);- final int payStart = last.payStart + (last.hasPayloads ? last.totalPositions : 0);- field =- new FieldData(- fieldNum, numTerms, positions, offsets, payloads, posStart, offStart, payStart);- }- fields.add(field);- return field;- }- }-- private DocData addDocData(int numVectorFields) {- FieldData last = null;- for (Iterator<DocData> it = pendingDocs.descendingIterator(); it.hasNext(); ) {- final DocData doc = it.next();- if (!doc.fields.isEmpty()) {- last = doc.fields.getLast();- break;- }- }- final DocData doc;- if (last == null) {- doc = new DocData(numVectorFields, 0, 0, 0);- } else {- final int posStart = last.posStart + (last.hasPositions ? last.totalPositions : 0);- final int offStart = last.offStart + (last.hasOffsets ? last.totalPositions : 0);- final int payStart = last.payStart + (last.hasPayloads ? last.totalPositions : 0);- doc = new DocData(numVectorFields, posStart, offStart, payStart);- }- pendingDocs.add(doc);- return doc;- }-- /** a pending field */- private class FieldData {- final boolean hasPositions, hasOffsets, hasPayloads;- final int fieldNum, flags, numTerms;- final int[] freqs, prefixLengths, suffixLengths;- final int posStart, offStart, payStart;- int totalPositions;- int ord;-- FieldData(- int fieldNum,- int numTerms,- boolean positions,- boolean offsets,- boolean payloads,- int posStart,- int offStart,- int payStart) {- this.fieldNum = fieldNum;- this.numTerms = numTerms;- this.hasPositions = positions;- this.hasOffsets = offsets;- this.hasPayloads = payloads;- this.flags =- (positions ? POSITIONS : 0) | (offsets ? OFFSETS : 0) | (payloads ? PAYLOADS : 0);- this.freqs = new int[numTerms];- this.prefixLengths = new int[numTerms];- this.suffixLengths = new int[numTerms];- this.posStart = posStart;- this.offStart = offStart;- this.payStart = payStart;- totalPositions = 0;- ord = 0;- }-- void addTerm(int freq, int prefixLength, int suffixLength) {- freqs[ord] = freq;- prefixLengths[ord] = prefixLength;- suffixLengths[ord] = suffixLength;- ++ord;- }-- void addPosition(int position, int startOffset, int length, int payloadLength) {- if (hasPositions) {- if (posStart + totalPositions == positionsBuf.length) {- positionsBuf = ArrayUtil.grow(positionsBuf);- }- positionsBuf[posStart + totalPositions] = position;- }- if (hasOffsets) {- if (offStart + totalPositions == startOffsetsBuf.length) {- final int newLength = ArrayUtil.oversize(offStart + totalPositions, 4);- startOffsetsBuf = ArrayUtil.growExact(startOffsetsBuf, newLength);- lengthsBuf = ArrayUtil.growExact(lengthsBuf, newLength);- }- startOffsetsBuf[offStart + totalPositions] = startOffset;- lengthsBuf[offStart + totalPositions] = length;- }- if (hasPayloads) {- if (payStart + totalPositions == payloadLengthsBuf.length) {- payloadLengthsBuf = ArrayUtil.grow(payloadLengthsBuf);- }- payloadLengthsBuf[payStart + totalPositions] = payloadLength;- }- ++totalPositions;- }- }-- private int numDocs; // total number of docs seen- private final Deque<DocData> pendingDocs; // pending docs- private DocData curDoc; // current document- private FieldData curField; // current field- private final BytesRef lastTerm;- private int[] positionsBuf, startOffsetsBuf, lengthsBuf, payloadLengthsBuf;- private final ByteBuffersDataOutput termSuffixes; // buffered term suffixes- private final ByteBuffersDataOutput payloadBytes; // buffered term payloads- private final BlockPackedWriter writer;- private final int maxDocsPerChunk; // hard limit on number of docs per chunk-- /** Sole constructor. */- Lucene50CompressingTermVectorsWriter(- Directory directory,- SegmentInfo si,- String segmentSuffix,- IOContext context,- String formatName,- CompressionMode compressionMode,- int chunkSize,- int maxDocsPerChunk,- int blockShift)- throws IOException {- assert directory != null;- this.segment = si.name;- this.compressor = compressionMode.newCompressor();- this.chunkSize = chunkSize;- this.maxDocsPerChunk = maxDocsPerChunk;-- numDocs = 0;- pendingDocs = new ArrayDeque<>();- termSuffixes = ByteBuffersDataOutput.newResettableInstance();- payloadBytes = ByteBuffersDataOutput.newResettableInstance();- lastTerm = new BytesRef(ArrayUtil.oversize(30, 1));-- boolean success = false;- try {- metaStream =- EndiannessReverserUtil.createOutput(- directory,- IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_META_EXTENSION),- context);- CodecUtil.writeIndexHeader(- metaStream,- VECTORS_INDEX_CODEC_NAME + "Meta",- VERSION_CURRENT,- si.getId(),- segmentSuffix);- assert CodecUtil.indexHeaderLength(VECTORS_INDEX_CODEC_NAME + "Meta", segmentSuffix)- == metaStream.getFilePointer();-- vectorsStream =- EndiannessReverserUtil.createOutput(- directory,- IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION),- context);- CodecUtil.writeIndexHeader(- vectorsStream, formatName, VERSION_CURRENT, si.getId(), segmentSuffix);- assert CodecUtil.indexHeaderLength(formatName, segmentSuffix)- == vectorsStream.getFilePointer();-- indexWriter =- new FieldsIndexWriter(- directory,- segment,- segmentSuffix,- VECTORS_INDEX_EXTENSION,- VECTORS_INDEX_CODEC_NAME,- si.getId(),- blockShift,- context);-- metaStream.writeVInt(PackedInts.VERSION_CURRENT);- metaStream.writeVInt(chunkSize);- writer = new BlockPackedWriter(vectorsStream, PACKED_BLOCK_SIZE);-- positionsBuf = new int[1024];- startOffsetsBuf = new int[1024];- lengthsBuf = new int[1024];- payloadLengthsBuf = new int[1024];-- success = true;- } finally {- if (!success) {- IOUtils.closeWhileHandlingException(metaStream, vectorsStream, indexWriter, indexWriter);- }- }- }-- @Override- public void close() throws IOException {- try {- IOUtils.close(metaStream, vectorsStream, indexWriter);- } finally {- metaStream = null;- vectorsStream = null;- indexWriter = null;- }- }-- @Override- public void startDocument(int numVectorFields) throws IOException {- curDoc = addDocData(numVectorFields);- }-- @Override- public void finishDocument() throws IOException {- // append the payload bytes of the doc after its terms- payloadBytes.copyTo(termSuffixes);- payloadBytes.reset();- ++numDocs;- if (triggerFlush()) {- flush();- }- curDoc = null;- }-- @Override- public void startField(- FieldInfo info, int numTerms, boolean positions, boolean offsets, boolean payloads)- throws IOException {- curField = curDoc.addField(info.number, numTerms, positions, offsets, payloads);- lastTerm.length = 0;- }-- @Override- public void finishField() throws IOException {- curField = null;- }-- @Override- public void startTerm(BytesRef term, int freq) throws IOException {- assert freq >= 1;- final int prefix;- if (lastTerm.length == 0) {- // no previous term: no bytes to write- prefix = 0;- } else {- prefix = StringHelper.bytesDifference(lastTerm, term);- }- curField.addTerm(freq, prefix, term.length - prefix);- termSuffixes.writeBytes(term.bytes, term.offset + prefix, term.length - prefix);- // copy last term- if (lastTerm.bytes.length < term.length) {- lastTerm.bytes = new byte[ArrayUtil.oversize(term.length, 1)];- }- lastTerm.offset = 0;- lastTerm.length = term.length;- System.arraycopy(term.bytes, term.offset, lastTerm.bytes, 0, term.length);- }-- @Override- public void addPosition(int position, int startOffset, int endOffset, BytesRef payload)- throws IOException {- assert curField.flags != 0;- curField.addPosition(- position, startOffset, endOffset - startOffset, payload == null ? 0 : payload.length);- if (curField.hasPayloads && payload != null) {- payloadBytes.writeBytes(payload.bytes, payload.offset, payload.length);- }- }-- private boolean triggerFlush() {- return termSuffixes.size() >= chunkSize || pendingDocs.size() >= maxDocsPerChunk;- }-- private void flush() throws IOException {- numChunks++;- final int chunkDocs = pendingDocs.size();- assert chunkDocs > 0 : chunkDocs;-- // write the index file- indexWriter.writeIndex(chunkDocs, vectorsStream.getFilePointer());-- final int docBase = numDocs - chunkDocs;- vectorsStream.writeVInt(docBase);- vectorsStream.writeVInt(chunkDocs);-- // total number of fields of the chunk- final int totalFields = flushNumFields(chunkDocs);-- if (totalFields > 0) {- // unique field numbers (sorted)- final int[] fieldNums = flushFieldNums();- // offsets in the array of unique field numbers- flushFields(totalFields, fieldNums);- // flags (does the field have positions, offsets, payloads?)- flushFlags(totalFields, fieldNums);- // number of terms of each field- flushNumTerms(totalFields);- // prefix and suffix lengths for each field- flushTermLengths();- // term freqs - 1 (because termFreq is always >=1) for each term- flushTermFreqs();- // positions for all terms, when enabled- flushPositions();- // offsets for all terms, when enabled- flushOffsets(fieldNums);- // payload lengths for all terms, when enabled- flushPayloadLengths();-- // compress terms and payloads and write them to the output- //- // TODO: We could compress in the slices we already have in the buffer (min/max slice- // can be set on the buffer itself).- byte[] content = termSuffixes.toArrayCopy();- compressor.compress(content, 0, content.length, vectorsStream);- }-- // reset- pendingDocs.clear();- curDoc = null;- curField = null;- termSuffixes.reset();- }-- private int flushNumFields(int chunkDocs) throws IOException {- if (chunkDocs == 1) {- final int numFields = pendingDocs.getFirst().numFields;- vectorsStream.writeVInt(numFields);- return numFields;- } else {- writer.reset(vectorsStream);- int totalFields = 0;- for (DocData dd : pendingDocs) {- writer.add(dd.numFields);- totalFields += dd.numFields;- }- writer.finish();- return totalFields;- }- }-- /** Returns a sorted array containing unique field numbers */- private int[] flushFieldNums() throws IOException {- SortedSet<Integer> fieldNums = new TreeSet<>();- for (DocData dd : pendingDocs) {- for (FieldData fd : dd.fields) {- fieldNums.add(fd.fieldNum);- }- }-- final int numDistinctFields = fieldNums.size();- assert numDistinctFields > 0;- final int bitsRequired = PackedInts.bitsRequired(fieldNums.last());- final int token = (Math.min(numDistinctFields - 1, 0x07) << 5) | bitsRequired;- vectorsStream.writeByte((byte) token);- if (numDistinctFields - 1 >= 0x07) {- vectorsStream.writeVInt(numDistinctFields - 1 - 0x07);- }- final PackedInts.Writer writer =- PackedInts.getWriterNoHeader(- vectorsStream, PackedInts.Format.PACKED, fieldNums.size(), bitsRequired, 1);- for (Integer fieldNum : fieldNums) {- writer.add(fieldNum);- }- writer.finish();-- int[] fns = new int[fieldNums.size()];- int i = 0;- for (Integer key : fieldNums) {- fns[i++] = key;- }- return fns;- }-- private void flushFields(int totalFields, int[] fieldNums) throws IOException {- final PackedInts.Writer writer =- PackedInts.getWriterNoHeader(- vectorsStream,- PackedInts.Format.PACKED,- totalFields,- PackedInts.bitsRequired(fieldNums.length - 1),- 1);- for (DocData dd : pendingDocs) {- for (FieldData fd : dd.fields) {- final int fieldNumIndex = Arrays.binarySearch(fieldNums, fd.fieldNum);- assert fieldNumIndex >= 0;- writer.add(fieldNumIndex);- }- }- writer.finish();- }-- private void flushFlags(int totalFields, int[] fieldNums) throws IOException {- // check if fields always have the same flags- boolean nonChangingFlags = true;- int[] fieldFlags = new int[fieldNums.length];- Arrays.fill(fieldFlags, -1);- outer:- for (DocData dd : pendingDocs) {- for (FieldData fd : dd.fields) {- final int fieldNumOff = Arrays.binarySearch(fieldNums, fd.fieldNum);- assert fieldNumOff >= 0;- if (fieldFlags[fieldNumOff] == -1) {- fieldFlags[fieldNumOff] = fd.flags;- } else if (fieldFlags[fieldNumOff] != fd.flags) {- nonChangingFlags = false;- break outer;- }- }- }-- if (nonChangingFlags) {- // write one flag per field num- vectorsStream.writeVInt(0);- final PackedInts.Writer writer =- PackedInts.getWriterNoHeader(- vectorsStream, PackedInts.Format.PACKED, fieldFlags.length, FLAGS_BITS, 1);- for (int flags : fieldFlags) {- assert flags >= 0;- writer.add(flags);- }- assert writer.ord() == fieldFlags.length - 1;- writer.finish();- } else {- // write one flag for every field instance- vectorsStream.writeVInt(1);- final PackedInts.Writer writer =- PackedInts.getWriterNoHeader(- vectorsStream, PackedInts.Format.PACKED, totalFields, FLAGS_BITS, 1);- for (DocData dd : pendingDocs) {- for (FieldData fd : dd.fields) {- writer.add(fd.flags);- }- }- assert writer.ord() == totalFields - 1;- writer.finish();- }- }-- private void flushNumTerms(int totalFields) throws IOException {- int maxNumTerms = 0;- for (DocData dd : pendingDocs) {- for (FieldData fd : dd.fields) {- maxNumTerms |= fd.numTerms;- }- }- final int bitsRequired = PackedInts.bitsRequired(maxNumTerms);- vectorsStream.writeVInt(bitsRequired);- final PackedInts.Writer writer =- PackedInts.getWriterNoHeader(- vectorsStream, PackedInts.Format.PACKED, totalFields, bitsRequired, 1);- for (DocData dd : pendingDocs) {- for (FieldData fd : dd.fields) {- writer.add(fd.numTerms);- }- }- assert writer.ord() == totalFields - 1;- writer.finish();- }-- private void flushTermLengths() throws IOException {- writer.reset(vectorsStream);- for (DocData dd : pendingDocs) {- for (FieldData fd : dd.fields) {- for (int i = 0; i < fd.numTerms; ++i) {- writer.add(fd.prefixLengths[i]);- }- }- }- writer.finish();- writer.reset(vectorsStream);- for (DocData dd : pendingDocs) {- for (FieldData fd : dd.fields) {- for (int i = 0; i < fd.numTerms; ++i) {- writer.add(fd.suffixLengths[i]);- }- }- }- writer.finish();- }-- private void flushTermFreqs() throws IOException {- writer.reset(vectorsStream);- for (DocData dd : pendingDocs) {- for (FieldData fd : dd.fields) {- for (int i = 0; i < fd.numTerms; ++i) {- writer.add(fd.freqs[i] - 1);- }- }- }- writer.finish();- }-- private void flushPositions() throws IOException {- writer.reset(vectorsStream);- for (DocData dd : pendingDocs) {- for (FieldData fd : dd.fields) {- if (fd.hasPositions) {- int pos = 0;- for (int i = 0; i < fd.numTerms; ++i) {- int previousPosition = 0;- for (int j = 0; j < fd.freqs[i]; ++j) {- final int position = positionsBuf[fd.posStart + pos++];- writer.add(position - previousPosition);- previousPosition = position;- }- }- assert pos == fd.totalPositions;- }- }- }- writer.finish();- }-- private void flushOffsets(int[] fieldNums) throws IOException {- boolean hasOffsets = false;- long[] sumPos = new long[fieldNums.length];- long[] sumOffsets = new long[fieldNums.length];- for (DocData dd : pendingDocs) {- for (FieldData fd : dd.fields) {- hasOffsets |= fd.hasOffsets;- if (fd.hasOffsets && fd.hasPositions) {- final int fieldNumOff = Arrays.binarySearch(fieldNums, fd.fieldNum);- int pos = 0;- for (int i = 0; i < fd.numTerms; ++i) {- sumPos[fieldNumOff] += positionsBuf[fd.posStart + fd.freqs[i] - 1 + pos];- sumOffsets[fieldNumOff] += startOffsetsBuf[fd.offStart + fd.freqs[i] - 1 + pos];- pos += fd.freqs[i];- }- assert pos == fd.totalPositions;- }- }- }-- if (!hasOffsets) {- // nothing to do- return;- }-- final float[] charsPerTerm = new float[fieldNums.length];- for (int i = 0; i < fieldNums.length; ++i) {- charsPerTerm[i] =- (sumPos[i] <= 0 || sumOffsets[i] <= 0) ? 0 : (float) ((double) sumOffsets[i] / sumPos[i]);- }-- // start offsets- for (int i = 0; i < fieldNums.length; ++i) {- vectorsStream.writeInt(Float.floatToRawIntBits(charsPerTerm[i]));- }-- writer.reset(vectorsStream);- for (DocData dd : pendingDocs) {- for (FieldData fd : dd.fields) {- if ((fd.flags & OFFSETS) != 0) {- final int fieldNumOff = Arrays.binarySearch(fieldNums, fd.fieldNum);- final float cpt = charsPerTerm[fieldNumOff];- int pos = 0;- for (int i = 0; i < fd.numTerms; ++i) {- int previousPos = 0;- int previousOff = 0;- for (int j = 0; j < fd.freqs[i]; ++j) {- final int position = fd.hasPositions ? positionsBuf[fd.posStart + pos] : 0;- final int startOffset = startOffsetsBuf[fd.offStart + pos];- writer.add(startOffset - previousOff - (int) (cpt * (position - previousPos)));- previousPos = position;- previousOff = startOffset;- ++pos;- }- }- }- }- }- writer.finish();-- // lengths- writer.reset(vectorsStream);- for (DocData dd : pendingDocs) {- for (FieldData fd : dd.fields) {- if ((fd.flags & OFFSETS) != 0) {- int pos = 0;- for (int i = 0; i < fd.numTerms; ++i) {- for (int j = 0; j < fd.freqs[i]; ++j) {- writer.add(- lengthsBuf[fd.offStart + pos++] - fd.prefixLengths[i] - fd.suffixLengths[i]);- }- }- assert pos == fd.totalPositions;- }- }- }- writer.finish();- }-- private void flushPayloadLengths() throws IOException {- writer.reset(vectorsStream);- for (DocData dd : pendingDocs) {- for (FieldData fd : dd.fields) {- if (fd.hasPayloads) {- for (int i = 0; i < fd.totalPositions; ++i) {- writer.add(payloadLengthsBuf[fd.payStart + i]);- }- }- }- }- writer.finish();- }-- @Override- public void finish(int numDocs) throws IOException {- if (!pendingDocs.isEmpty()) {- numDirtyChunks++; // incomplete: we had to force this flush- numDirtyDocs += pendingDocs.size();- flush();- }- if (numDocs != this.numDocs) {- throw new RuntimeException(- "Wrote " + this.numDocs + " docs, finish called with numDocs=" + numDocs);- }- indexWriter.finish(numDocs, vectorsStream.getFilePointer(), metaStream);- metaStream.writeVLong(numChunks);- metaStream.writeVLong(numDirtyChunks);- metaStream.writeVLong(numDirtyDocs);- CodecUtil.writeFooter(metaStream);- CodecUtil.writeFooter(vectorsStream);- }-- @Override- public void addProx(int numProx, DataInput positions, DataInput offsets) throws IOException {- assert (curField.hasPositions) == (positions != null);- assert (curField.hasOffsets) == (offsets != null);-- if (curField.hasPositions) {- final int posStart = curField.posStart + curField.totalPositions;- if (posStart + numProx > positionsBuf.length) {- positionsBuf = ArrayUtil.grow(positionsBuf, posStart + numProx);- }- int position = 0;- if (curField.hasPayloads) {- final int payStart = curField.payStart + curField.totalPositions;- if (payStart + numProx > payloadLengthsBuf.length) {- payloadLengthsBuf = ArrayUtil.grow(payloadLengthsBuf, payStart + numProx);- }- for (int i = 0; i < numProx; ++i) {- final int code = positions.readVInt();- if ((code & 1) != 0) {- // This position has a payload- final int payloadLength = positions.readVInt();- payloadLengthsBuf[payStart + i] = payloadLength;- payloadBytes.copyBytes(positions, payloadLength);- } else {- payloadLengthsBuf[payStart + i] = 0;- }- position += code >>> 1;- positionsBuf[posStart + i] = position;- }- } else {- for (int i = 0; i < numProx; ++i) {- position += (positions.readVInt() >>> 1);- positionsBuf[posStart + i] = position;- }- }- }-- if (curField.hasOffsets) {- final int offStart = curField.offStart + curField.totalPositions;- if (offStart + numProx > startOffsetsBuf.length) {- final int newLength = ArrayUtil.oversize(offStart + numProx, 4);- startOffsetsBuf = ArrayUtil.growExact(startOffsetsBuf, newLength);- lengthsBuf = ArrayUtil.growExact(lengthsBuf, newLength);- }- int lastOffset = 0, startOffset, endOffset;- for (int i = 0; i < numProx; ++i) {- startOffset = lastOffset + offsets.readVInt();- endOffset = startOffset + offsets.readVInt();- lastOffset = endOffset;- startOffsetsBuf[offStart + i] = startOffset;- lengthsBuf[offStart + i] = endOffset - startOffset;- }- }-- curField.totalPositions += numProx;- }-- // bulk merge is scary: its caused corruption bugs in the past.- // we try to be extra safe with this impl, but add an escape hatch to- // have a workaround for undiscovered bugs.- static final String BULK_MERGE_ENABLED_SYSPROP =- Lucene50CompressingTermVectorsWriter.class.getName() + ".enableBulkMerge";- static final boolean BULK_MERGE_ENABLED;-- static {- boolean v = true;- try {- v = Boolean.parseBoolean(System.getProperty(BULK_MERGE_ENABLED_SYSPROP, "true"));- } catch (SecurityException _) {- }- BULK_MERGE_ENABLED = v;- }-- @Override- public long ramBytesUsed() {- return positionsBuf.length- + startOffsetsBuf.length- + lengthsBuf.length- + payloadLengthsBuf.length- + termSuffixes.ramBytesUsed()- + payloadBytes.ramBytesUsed()- + lastTerm.bytes.length;- }-- @Override- public Collection<Accountable> getChildResources() {- return List.of(termSuffixes, payloadBytes);- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50RWCompressingStoredFieldsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50RWCompressingStoredFieldsFormat.javadeleted file mode 100644index 6983944ed5e..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50RWCompressingStoredFieldsFormat.java+++ /dev/null@@ -1,52 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50.compressing;--import java.io.IOException;-import org.apache.lucene.backward_codecs.compressing.CompressionMode;-import org.apache.lucene.codecs.StoredFieldsWriter;-import org.apache.lucene.index.SegmentInfo;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;--/** RW impersonation of Lucene50CompressingStoredFieldsFormat. */-public class Lucene50RWCompressingStoredFieldsFormat extends Lucene50CompressingStoredFieldsFormat {-- public Lucene50RWCompressingStoredFieldsFormat(- String formatName,- CompressionMode compressionMode,- int chunkSize,- int maxDocsPerChunk,- int blockShift) {- super(formatName, compressionMode, chunkSize, maxDocsPerChunk, blockShift);- }-- @Override- public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context)- throws IOException {- return new Lucene50CompressingStoredFieldsWriter(- directory,- si,- segmentSuffix,- context,- formatName,- compressionMode,- chunkSize,- maxDocsPerChunk,- blockShift);- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50RWCompressingTermVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50RWCompressingTermVectorsFormat.javadeleted file mode 100644index f10b1418413..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50RWCompressingTermVectorsFormat.java+++ /dev/null@@ -1,54 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene50.compressing;--import java.io.IOException;-import org.apache.lucene.backward_codecs.compressing.CompressionMode;-import org.apache.lucene.codecs.TermVectorsWriter;-import org.apache.lucene.index.SegmentInfo;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;--/** RW impersonation of Lucene50CompressingTermVectorsFormat. */-public class Lucene50RWCompressingTermVectorsFormat extends Lucene50CompressingTermVectorsFormat {-- /** Sole constructor. */- public Lucene50RWCompressingTermVectorsFormat(- String formatName,- String segmentSuffix,- CompressionMode compressionMode,- int chunkSize,- int maxDocsPerChunk,- int blockSize) {- super(formatName, segmentSuffix, compressionMode, chunkSize, maxDocsPerChunk, blockSize);- }-- @Override- public final TermVectorsWriter vectorsWriter(- Directory directory, SegmentInfo segmentInfo, IOContext context) throws IOException {- return new Lucene50CompressingTermVectorsWriter(- directory,- segmentInfo,- segmentSuffix,- context,- formatName,- compressionMode,- chunkSize,- maxDocsPerChunk,- blockSize);- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/Lucene60PointsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/Lucene60PointsWriter.javadeleted file mode 100644index 62f227c699e..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/Lucene60PointsWriter.java+++ /dev/null@@ -1,286 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene60;--import java.io.IOException;-import java.util.ArrayList;-import java.util.HashMap;-import java.util.List;-import java.util.Map;-import org.apache.lucene.backward_codecs.lucene60.bkd.BKDWriter60;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.MutablePointTree;-import org.apache.lucene.codecs.PointsReader;-import org.apache.lucene.codecs.PointsWriter;-import org.apache.lucene.index.FieldInfo;-import org.apache.lucene.index.FieldInfos;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.MergeState;-import org.apache.lucene.index.PointValues;-import org.apache.lucene.index.PointValues.IntersectVisitor;-import org.apache.lucene.index.PointValues.Relation;-import org.apache.lucene.index.SegmentWriteState;-import org.apache.lucene.store.IndexOutput;-import org.apache.lucene.util.IOUtils;--/** Writes dimensional values */-public class Lucene60PointsWriter extends PointsWriter {-- /** Output used to write the BKD tree data file */- protected final IndexOutput dataOut;-- /** Maps field name to file pointer in the data file where the BKD index is located. */- protected final Map<String, Long> indexFPs = new HashMap<>();-- final SegmentWriteState writeState;- final int maxPointsInLeafNode;- final double maxMBSortInHeap;- private boolean finished;-- /** Full constructor */- public Lucene60PointsWriter(- SegmentWriteState writeState, int maxPointsInLeafNode, double maxMBSortInHeap)- throws IOException {- assert writeState.fieldInfos.hasPointValues();- this.writeState = writeState;- this.maxPointsInLeafNode = maxPointsInLeafNode;- this.maxMBSortInHeap = maxMBSortInHeap;- String dataFileName =- IndexFileNames.segmentFileName(- writeState.segmentInfo.name,- writeState.segmentSuffix,- Lucene60PointsFormat.DATA_EXTENSION);- dataOut =- EndiannessReverserUtil.createOutput(writeState.directory, dataFileName, writeState.context);- boolean success = false;- try {- CodecUtil.writeIndexHeader(- dataOut,- Lucene60PointsFormat.DATA_CODEC_NAME,- Lucene60PointsFormat.DATA_VERSION_CURRENT,- writeState.segmentInfo.getId(),- writeState.segmentSuffix);- success = true;- } finally {- if (success == false) {- IOUtils.closeWhileHandlingException(dataOut);- }- }- }-- /**- * Uses the defaults values for {@code maxPointsInLeafNode} (1024) and {@code maxMBSortInHeap}- * (16.0)- */- public Lucene60PointsWriter(SegmentWriteState writeState) throws IOException {- this(- writeState,- BKDWriter60.DEFAULT_MAX_POINTS_IN_LEAF_NODE,- BKDWriter60.DEFAULT_MAX_MB_SORT_IN_HEAP);- }-- @Override- public void writeField(FieldInfo fieldInfo, PointsReader reader) throws IOException {-- PointValues.PointTree values = reader.getValues(fieldInfo.name).getPointTree();-- try (BKDWriter60 writer =- new BKDWriter60(- writeState.segmentInfo.maxDoc(),- writeState.directory,- writeState.segmentInfo.name,- fieldInfo.getPointDimensionCount(),- fieldInfo.getPointIndexDimensionCount(),- fieldInfo.getPointNumBytes(),- maxPointsInLeafNode,- maxMBSortInHeap,- values.size())) {-- if (values instanceof MutablePointTree) {- final long fp = writer.writeField(dataOut, fieldInfo.name, (MutablePointTree) values);- if (fp != -1) {- indexFPs.put(fieldInfo.name, fp);- }- return;- }-- values.visitDocValues(- new IntersectVisitor() {- @Override- public void visit(int docID) {- throw new IllegalStateException();- }-- @Override- public void visit(int docID, byte[] packedValue) throws IOException {- writer.add(packedValue, docID);- }-- @Override- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {- return Relation.CELL_CROSSES_QUERY;- }- });-- // We could have 0 points on merge since all docs with dimensional fields may be deleted:- if (writer.getPointCount() > 0) {- indexFPs.put(fieldInfo.name, writer.finish(dataOut));- }- }- }-- @Override- public void merge(MergeState mergeState) throws IOException {- /*- * If indexSort is activated and some of the leaves are not sorted the next test will catch that- * and the non-optimized merge will run. If the readers are all sorted then it's safe to perform- * a bulk merge of the points.- */- for (PointsReader reader : mergeState.pointsReaders) {- if (reader instanceof Lucene60PointsReader == false) {- // We can only bulk merge when all to-be-merged segments use our format:- super.merge(mergeState);- return;- }- }- for (PointsReader reader : mergeState.pointsReaders) {- if (reader != null) {- reader.checkIntegrity();- }- }-- for (FieldInfo fieldInfo : mergeState.mergeFieldInfos) {- if (fieldInfo.getPointDimensionCount() != 0) {- if (fieldInfo.getPointDimensionCount() == 1) {-- // Worst case total maximum size (if none of the points are deleted):- long totMaxSize = 0;- for (int i = 0; i < mergeState.pointsReaders.length; i++) {- PointsReader reader = mergeState.pointsReaders[i];- if (reader != null) {- FieldInfos readerFieldInfos = mergeState.fieldInfos[i];- FieldInfo readerFieldInfo = readerFieldInfos.fieldInfo(fieldInfo.name);- if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {- PointValues values = reader.getValues(fieldInfo.name);- if (values != null) {- totMaxSize += values.size();- }- }- }- }-- // System.out.println("MERGE: field=" + fieldInfo.name);- // Optimize the 1D case to use BKDWriter.merge, which does a single merge sort of the- // already sorted incoming segments, instead of trying to sort all points again as if- // we were simply reindexing them:- try (BKDWriter60 writer =- new BKDWriter60(- writeState.segmentInfo.maxDoc(),- writeState.directory,- writeState.segmentInfo.name,- fieldInfo.getPointDimensionCount(),- fieldInfo.getPointIndexDimensionCount(),- fieldInfo.getPointNumBytes(),- maxPointsInLeafNode,- maxMBSortInHeap,- totMaxSize)) {- List<PointValues> bkdReaders = new ArrayList<>();- List<MergeState.DocMap> docMaps = new ArrayList<>();- for (int i = 0; i < mergeState.pointsReaders.length; i++) {- PointsReader reader = mergeState.pointsReaders[i];-- if (reader != null) {-- // we confirmed this up above- assert reader instanceof Lucene60PointsReader;- Lucene60PointsReader reader60 = (Lucene60PointsReader) reader;-- // NOTE: we cannot just use the merged fieldInfo.number (instead of resolving to- // this- // reader's FieldInfo as we do below) because field numbers can easily be different- // when addIndexes(Directory...) copies over segments from another index:-- FieldInfos readerFieldInfos = mergeState.fieldInfos[i];- FieldInfo readerFieldInfo = readerFieldInfos.fieldInfo(fieldInfo.name);- if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {- PointValues bkdReader = reader60.getValues(readerFieldInfo.name);- if (bkdReader != null) {- bkdReaders.add(bkdReader);- docMaps.add(mergeState.docMaps[i]);- }- }- }- }-- long fp = writer.merge(dataOut, docMaps, bkdReaders);- if (fp != -1) {- indexFPs.put(fieldInfo.name, fp);- }- }- } else {- mergeOneField(mergeState, fieldInfo);- }- }- }-- finish();- }-- @Override- public void finish() throws IOException {- if (finished) {- throw new IllegalStateException("already finished");- }- finished = true;- CodecUtil.writeFooter(dataOut);-- String indexFileName =- IndexFileNames.segmentFileName(- writeState.segmentInfo.name,- writeState.segmentSuffix,- Lucene60PointsFormat.INDEX_EXTENSION);- // Write index file- try (IndexOutput indexOut =- EndiannessReverserUtil.createOutput(- writeState.directory, indexFileName, writeState.context)) {- CodecUtil.writeIndexHeader(- indexOut,- Lucene60PointsFormat.META_CODEC_NAME,- Lucene60PointsFormat.INDEX_VERSION_CURRENT,- writeState.segmentInfo.getId(),- writeState.segmentSuffix);- int count = indexFPs.size();- indexOut.writeVInt(count);- for (Map.Entry<String, Long> ent : indexFPs.entrySet()) {- FieldInfo fieldInfo = writeState.fieldInfos.fieldInfo(ent.getKey());- if (fieldInfo == null) {- throw new IllegalStateException(- "wrote field=\"" + ent.getKey() + "\" but that field doesn't exist in FieldInfos");- }- indexOut.writeVInt(fieldInfo.number);- indexOut.writeVLong(ent.getValue());- }- CodecUtil.writeFooter(indexOut);- }- }-- @Override- public void close() throws IOException {- dataOut.close();- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/Lucene60RWPointsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/Lucene60RWPointsFormat.javadeleted file mode 100644index 3df9edd7360..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/Lucene60RWPointsFormat.java+++ /dev/null@@ -1,33 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene60;--import java.io.IOException;-import org.apache.lucene.codecs.PointsWriter;-import org.apache.lucene.index.SegmentWriteState;--/** RW variant of Lucene60PointsFormat */-public class Lucene60RWPointsFormat extends Lucene60PointsFormat {-- /** Sole constructor. */- public Lucene60RWPointsFormat() {}-- @Override- public PointsWriter fieldsWriter(SegmentWriteState state) throws IOException {- return new Lucene60PointsWriter(state);- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/TestLucene60FieldInfosFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/TestLucene60FieldInfosFormat.javadeleted file mode 100644index 0b9e9f72672..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/TestLucene60FieldInfosFormat.java+++ /dev/null@@ -1,35 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene60;--import org.apache.lucene.backward_codecs.lucene84.Lucene84RWCodec;-import org.apache.lucene.codecs.Codec;-import org.apache.lucene.tests.index.BaseFieldInfoFormatTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;--@Nightly // N-2 formats are only tested on nightly runs-public class TestLucene60FieldInfosFormat extends BaseFieldInfoFormatTestCase {- @Override- protected Codec getCodec() {- return new Lucene84RWCodec();- }-- @Override- protected boolean supportDocValuesSkipIndex() {- return false;- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/TestLucene60PointsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/TestLucene60PointsFormat.javadeleted file mode 100644index ee1cd9f66bb..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/TestLucene60PointsFormat.java+++ /dev/null@@ -1,320 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene60;--import java.io.IOException;-import java.util.Arrays;-import org.apache.lucene.backward_codecs.lucene60.bkd.BKDWriter60;-import org.apache.lucene.backward_codecs.lucene84.Lucene84RWCodec;-import org.apache.lucene.codecs.Codec;-import org.apache.lucene.document.BinaryPoint;-import org.apache.lucene.document.Document;-import org.apache.lucene.index.DirectoryReader;-import org.apache.lucene.index.IndexReader;-import org.apache.lucene.index.IndexWriter;-import org.apache.lucene.index.IndexWriterConfig;-import org.apache.lucene.index.LeafReader;-import org.apache.lucene.index.PointValues;-import org.apache.lucene.index.PointValues.IntersectVisitor;-import org.apache.lucene.index.PointValues.Relation;-import org.apache.lucene.store.Directory;-import org.apache.lucene.tests.index.BasePointsFormatTestCase;-import org.apache.lucene.tests.index.MockRandomMergePolicy;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;-import org.apache.lucene.tests.util.TestUtil;--/** Tests Lucene60PointsFormat */-@Nightly // N-2 formats are only tested on nightly runs-public class TestLucene60PointsFormat extends BasePointsFormatTestCase {- private final Codec codec;- private final int maxPointsInLeafNode;-- public TestLucene60PointsFormat() {- codec = new Lucene84RWCodec();- maxPointsInLeafNode = BKDWriter60.DEFAULT_MAX_POINTS_IN_LEAF_NODE;- }-- @Override- protected Codec getCodec() {- return codec;- }-- // TODO: clean up the math/estimation here rather than suppress so many warnings- @SuppressWarnings({"NarrowCalculation", "LongDoubleConversion"})- public void testEstimatePointCount() throws IOException {- Directory dir = newDirectory();- IndexWriterConfig iwc = newIndexWriterConfig();- // Avoid mockRandomMP since it may cause non-optimal merges that make the- // number of points per leaf hard to predict- while (iwc.getMergePolicy() instanceof MockRandomMergePolicy) {- iwc.setMergePolicy(newMergePolicy());- }- IndexWriter w = new IndexWriter(dir, iwc);- byte[] pointValue = new byte[3];- byte[] uniquePointValue = new byte[3];- random().nextBytes(uniquePointValue);- final int numDocs =- TEST_NIGHTLY ? atLeast(10000) : atLeast(500); // at night, make sure we have several leaves- final boolean multiValues = random().nextBoolean();- int totalValues = 0;- for (int i = 0; i < numDocs; ++i) {- Document doc = new Document();- if (i == numDocs / 2) {- totalValues++;- doc.add(new BinaryPoint("f", uniquePointValue));- } else {- final int numValues = (multiValues) ? TestUtil.nextInt(random(), 2, 100) : 1;- for (int j = 0; j < numValues; j++) {- do {- random().nextBytes(pointValue);- } while (Arrays.equals(pointValue, uniquePointValue));- doc.add(new BinaryPoint("f", pointValue));- totalValues++;- }- }- w.addDocument(doc);- }- w.forceMerge(1);- final IndexReader r = DirectoryReader.open(w);- w.close();- final LeafReader lr = getOnlyLeafReader(r);- PointValues points = lr.getPointValues("f");-- IntersectVisitor allPointsVisitor =- new IntersectVisitor() {- @Override- public void visit(int docID, byte[] packedValue) throws IOException {}-- @Override- public void visit(int docID) throws IOException {}-- @Override- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {- return Relation.CELL_INSIDE_QUERY;- }- };-- assertEquals(totalValues, points.estimatePointCount(allPointsVisitor));- assertEquals(numDocs, points.estimateDocCount(allPointsVisitor));-- IntersectVisitor noPointsVisitor =- new IntersectVisitor() {- @Override- public void visit(int docID, byte[] packedValue) throws IOException {}-- @Override- public void visit(int docID) throws IOException {}-- @Override- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {- return Relation.CELL_OUTSIDE_QUERY;- }- };-- // Return 0 if no points match- assertEquals(0, points.estimatePointCount(noPointsVisitor));- assertEquals(0, points.estimateDocCount(noPointsVisitor));-- IntersectVisitor onePointMatchVisitor =- new IntersectVisitor() {- @Override- public void visit(int docID, byte[] packedValue) throws IOException {}-- @Override- public void visit(int docID) throws IOException {}-- @Override- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {- if (Arrays.compareUnsigned(uniquePointValue, 0, 3, maxPackedValue, 0, 3) > 0- || Arrays.compareUnsigned(uniquePointValue, 0, 3, minPackedValue, 0, 3) < 0) {- return Relation.CELL_OUTSIDE_QUERY;- }- return Relation.CELL_CROSSES_QUERY;- }- };-- // If only one point matches, then the point count is (maxPointsInLeafNode + 1) / 2- // in general, or maybe 2x that if the point is a split value- final long pointCount = points.estimatePointCount(onePointMatchVisitor);- final long lastNodePointCount = totalValues % maxPointsInLeafNode;- assertTrue(- "" + pointCount,- pointCount == (maxPointsInLeafNode + 1) / 2 // common case- || pointCount == (lastNodePointCount + 1) / 2 // not fully populated leaf- || pointCount == 2 * ((maxPointsInLeafNode + 1) / 2) // if the point is a split value- || pointCount- == ((maxPointsInLeafNode + 1) / 2)- + ((lastNodePointCount + 1)- / 2)); // if the point is a split value and one leaf is not fully populated-- final long docCount = points.estimateDocCount(onePointMatchVisitor);-- if (multiValues) {- assertEquals(- docCount,- (long)- (docCount- * (1d- - Math.pow(- (numDocs - pointCount) / points.size(), points.size() / docCount))));- } else {- assertEquals(Math.min(pointCount, numDocs), docCount);- }- r.close();- dir.close();- }-- // The tree is always balanced in the N dims case, and leaves are- // not all full so things are a bit different- // TODO: clean up the math/estimation here rather than suppress so many warnings- @SuppressWarnings({"NarrowCalculation", "LongDoubleConversion"})- public void testEstimatePointCount2Dims() throws IOException {- Directory dir = newDirectory();- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());- byte[][] pointValue = new byte[2][];- pointValue[0] = new byte[3];- pointValue[1] = new byte[3];- byte[][] uniquePointValue = new byte[2][];- uniquePointValue[0] = new byte[3];- uniquePointValue[1] = new byte[3];- random().nextBytes(uniquePointValue[0]);- random().nextBytes(uniquePointValue[1]);- final int numDocs =- TEST_NIGHTLY- ? atLeast(10000)- : atLeast(1000); // in nightly, make sure we have several leaves- final boolean multiValues = random().nextBoolean();- int totalValues = 0;- for (int i = 0; i < numDocs; ++i) {- Document doc = new Document();- if (i == numDocs / 2) {- doc.add(new BinaryPoint("f", uniquePointValue));- totalValues++;- } else {- final int numValues = (multiValues) ? TestUtil.nextInt(random(), 2, 100) : 1;- for (int j = 0; j < numValues; j++) {- do {- random().nextBytes(pointValue[0]);- random().nextBytes(pointValue[1]);- } while (Arrays.equals(pointValue[0], uniquePointValue[0])- || Arrays.equals(pointValue[1], uniquePointValue[1]));- doc.add(new BinaryPoint("f", pointValue));- totalValues++;- }- }- w.addDocument(doc);- }- w.forceMerge(1);- final IndexReader r = DirectoryReader.open(w);- w.close();- final LeafReader lr = getOnlyLeafReader(r);- PointValues points = lr.getPointValues("f");-- IntersectVisitor allPointsVisitor =- new IntersectVisitor() {- @Override- public void visit(int docID, byte[] packedValue) throws IOException {}-- @Override- public void visit(int docID) throws IOException {}-- @Override- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {- return Relation.CELL_INSIDE_QUERY;- }- };-- assertEquals(totalValues, points.estimatePointCount(allPointsVisitor));- assertEquals(numDocs, points.estimateDocCount(allPointsVisitor));-- IntersectVisitor noPointsVisitor =- new IntersectVisitor() {- @Override- public void visit(int docID, byte[] packedValue) throws IOException {}-- @Override- public void visit(int docID) throws IOException {}-- @Override- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {- return Relation.CELL_OUTSIDE_QUERY;- }- };-- // Return 0 if no points match- assertEquals(0, points.estimatePointCount(noPointsVisitor));- assertEquals(0, points.estimateDocCount(noPointsVisitor));-- IntersectVisitor onePointMatchVisitor =- new IntersectVisitor() {- @Override- public void visit(int docID, byte[] packedValue) throws IOException {}-- @Override- public void visit(int docID) throws IOException {}-- @Override- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {- for (int dim = 0; dim < 2; ++dim) {- if (Arrays.compareUnsigned(- uniquePointValue[dim], 0, 3, maxPackedValue, dim * 3, dim * 3 + 3)- > 0- || Arrays.compareUnsigned(- uniquePointValue[dim], 0, 3, minPackedValue, dim * 3, dim * 3 + 3)- < 0) {- return Relation.CELL_OUTSIDE_QUERY;- }- }- return Relation.CELL_CROSSES_QUERY;- }- };-- final long pointCount = points.estimatePointCount(onePointMatchVisitor);- // With >1 dims, the tree is balanced- long actualMaxPointsInLeafNode = points.size();- while (actualMaxPointsInLeafNode > maxPointsInLeafNode) {- actualMaxPointsInLeafNode = (actualMaxPointsInLeafNode + 1) / 2;- }- final long countPerFullLeaf = (actualMaxPointsInLeafNode + 1) / 2;- final long countPerNotFullLeaf = (actualMaxPointsInLeafNode) / 2;- assertTrue(- pointCount + " vs " + actualMaxPointsInLeafNode,- // common case, point in one leaf.- pointCount >= countPerNotFullLeaf && pointCount <= countPerFullLeaf- ||- // one dimension is a split value- pointCount >= 2 * countPerNotFullLeaf && pointCount <= 2 * countPerFullLeaf- ||- // both dimensions are split values- pointCount >= 4 * countPerNotFullLeaf && pointCount <= 4 * countPerFullLeaf);-- final long docCount = points.estimateDocCount(onePointMatchVisitor);- if (multiValues) {- assertEquals(- docCount,- (long)- (docCount- * (1d- - Math.pow(- (numDocs - pointCount) / points.size(), points.size() / docCount))));- } else {- assertEquals(Math.min(pointCount, numDocs), docCount);- }- r.close();- dir.close();- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/bkd/BKDWriter60.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/bkd/BKDWriter60.javadeleted file mode 100644index c3381eff6b1..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/bkd/BKDWriter60.java+++ /dev/null@@ -1,2291 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene60.bkd;--import java.io.Closeable;-import java.io.IOException;-import java.util.ArrayList;-import java.util.Arrays;-import java.util.Comparator;-import java.util.List;-import java.util.function.IntFunction;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.MutablePointTree;-import org.apache.lucene.index.MergeState;-import org.apache.lucene.index.PointValues;-import org.apache.lucene.index.PointValues.IntersectVisitor;-import org.apache.lucene.index.PointValues.Relation;-import org.apache.lucene.store.ByteBuffersDataOutput;-import org.apache.lucene.store.ChecksumIndexInput;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IndexOutput;-import org.apache.lucene.store.TrackingDirectoryWrapper;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.BytesRefBuilder;-import org.apache.lucene.util.FixedBitSet;-import org.apache.lucene.util.IOUtils;-import org.apache.lucene.util.NumericUtils;-import org.apache.lucene.util.PriorityQueue;-import org.apache.lucene.util.bkd.BKDConfig;-import org.apache.lucene.util.bkd.BKDRadixSelector;-import org.apache.lucene.util.bkd.BKDReader;-import org.apache.lucene.util.bkd.HeapPointWriter;-import org.apache.lucene.util.bkd.MutablePointTreeReaderUtils;-import org.apache.lucene.util.bkd.OfflinePointWriter;-import org.apache.lucene.util.bkd.PointReader;-import org.apache.lucene.util.bkd.PointValue;-import org.apache.lucene.util.bkd.PointWriter;--// TODO-// - allow variable length byte[] (across docs and dims), but this is quite a bit more hairy-// - we could also index "auto-prefix terms" here, and use better compression, and maybe only use-// for the "fully contained" case so we'd-// only index docIDs-// - the index could be efficiently encoded as an FST, so we don't have wasteful-// (monotonic) long[] leafBlockFPs; or we could use MonotonicLongValues ... but then-// the index is already plenty small: 60M OSM points --> 1.1 MB with 128 points-// per leaf, and you can reduce that by putting more points per leaf-// - we could use threads while building; the higher nodes are very parallelizable--/**- * Recursively builds a block KD-tree to assign all incoming points in N-dim space to smaller and- * smaller N-dim rectangles (cells) until the number of points in a given rectangle is <= <code>- * maxPointsInLeafNode</code>. The tree is fully balanced, which means the leaf nodes will have- * between 50% and 100% of the requested <code>maxPointsInLeafNode</code>. Values that fall exactly- * on a cell boundary may be in either cell.- *- * <p>The number of dimensions can be 1 to 8, but every byte[] value is fixed length.- *- * <p>This consumes heap during writing: it allocates a <code>Long[numLeaves]</code>, a <code>- * byte[numLeaves*(1+bytesPerDim)]</code> and then uses up to the specified {@code maxMBSortInHeap}- * heap space for writing.- *- * <p><b>NOTE</b>: This can write at most Integer.MAX_VALUE * <code>maxPointsInLeafNode</code> /- * (1+bytesPerDim) total points.- *- * @lucene.experimental- */-public class BKDWriter60 implements Closeable {-- public static final String CODEC_NAME = "BKD";- public static final int VERSION_START = 4; // version used by Lucene 7.0- // public static final int VERSION_CURRENT = VERSION_START;- public static final int VERSION_LEAF_STORES_BOUNDS = 5;- public static final int VERSION_SELECTIVE_INDEXING = 6;- public static final int VERSION_LOW_CARDINALITY_LEAVES = 7;- public static final int VERSION_CURRENT = VERSION_LOW_CARDINALITY_LEAVES;-- /** How many bytes each docs takes in the fixed-width offline format */- private final int bytesPerDoc;-- /** Default maximum number of point in each leaf block */- public static final int DEFAULT_MAX_POINTS_IN_LEAF_NODE = 1024;-- /** Default maximum heap to use, before spilling to (slower) disk */- public static final float DEFAULT_MAX_MB_SORT_IN_HEAP = 16.0f;-- /** Maximum number of index dimensions (2 * max index dimensions) */- public static final int MAX_DIMS = 16;-- /** Maximum number of index dimensions */- public static final int MAX_INDEX_DIMS = 8;-- /** Number of splits before we compute the exact bounding box of an inner node. */- private static final int SPLITS_BEFORE_EXACT_BOUNDS = 4;-- /** How many dimensions we are storing at the leaf (data) nodes */- protected final int numDataDims;-- /** How many dimensions we are indexing in the internal nodes */- protected final int numIndexDims;-- /** How many bytes each value in each dimension takes. */- protected final int bytesPerDim;-- /** numDataDims * bytesPerDim */- protected final int packedBytesLength;-- /** numIndexDims * bytesPerDim */- protected final int packedIndexBytesLength;-- final TrackingDirectoryWrapper tempDir;- final String tempFileNamePrefix;- final double maxMBSortInHeap;-- final byte[] scratchDiff;- final byte[] scratch1;- final byte[] scratch2;- final BytesRef scratchBytesRef1 = new BytesRef();- final BytesRef scratchBytesRef2 = new BytesRef();- final int[] commonPrefixLengths;-- protected final FixedBitSet docsSeen;-- private PointWriter pointWriter;- private boolean finished;-- private IndexOutput tempInput;- protected final int maxPointsInLeafNode;- private final int maxPointsSortInHeap;-- /** Minimum per-dim values, packed */- protected final byte[] minPackedValue;-- /** Maximum per-dim values, packed */- protected final byte[] maxPackedValue;-- protected long pointCount;-- /** An upper bound on how many points the caller will add (includes deletions) */- private final long totalPointCount;-- private final int maxDoc;-- private final BKDConfig config;-- public BKDWriter60(- int maxDoc,- Directory tempDir,- String tempFileNamePrefix,- int numDataDims,- int numIndexDims,- int bytesPerDim,- int maxPointsInLeafNode,- double maxMBSortInHeap,- long totalPointCount)- throws IOException {- verifyParams(numDataDims, numIndexDims, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount);- // We use tracking dir to deal with removing files on exception, so each place that- // creates temp files doesn't need crazy try/finally/sucess logic:- this.tempDir = new TrackingDirectoryWrapper(tempDir);- this.tempFileNamePrefix = tempFileNamePrefix;- this.maxPointsInLeafNode = maxPointsInLeafNode;- this.numDataDims = numDataDims;- this.numIndexDims = numIndexDims;- this.bytesPerDim = bytesPerDim;- this.config = new BKDConfig(numDataDims, numIndexDims, bytesPerDim, maxPointsInLeafNode);- this.totalPointCount = totalPointCount;- this.maxDoc = maxDoc;- docsSeen = new FixedBitSet(maxDoc);- packedBytesLength = numDataDims * bytesPerDim;- packedIndexBytesLength = numIndexDims * bytesPerDim;-- scratchDiff = new byte[bytesPerDim];- scratch1 = new byte[packedBytesLength];- scratch2 = new byte[packedBytesLength];- commonPrefixLengths = new int[numDataDims];-- minPackedValue = new byte[packedIndexBytesLength];- maxPackedValue = new byte[packedIndexBytesLength];-- // dimensional values (numDims * bytesPerDim) + docID (int)- bytesPerDoc = packedBytesLength + Integer.BYTES;-- // Maximum number of points we hold in memory at any time- maxPointsSortInHeap = (int) ((maxMBSortInHeap * 1024 * 1024) / (bytesPerDoc));-- // Finally, we must be able to hold at least the leaf node in heap during build:- if (maxPointsSortInHeap < maxPointsInLeafNode) {- throw new IllegalArgumentException(- "maxMBSortInHeap="- + maxMBSortInHeap- + " only allows for maxPointsSortInHeap="- + maxPointsSortInHeap- + ", but this is less than maxPointsInLeafNode="- + maxPointsInLeafNode- + "; either increase maxMBSortInHeap or decrease maxPointsInLeafNode");- }-- this.maxMBSortInHeap = maxMBSortInHeap;- }-- public static void verifyParams(- int numDims,- int numIndexDims,- int maxPointsInLeafNode,- double maxMBSortInHeap,- long totalPointCount) {- // We encode dim in a single byte in the splitPackedValues, but we only expose 4 bits for it- // now, in case we want to use- // remaining 4 bits for another purpose later- if (numDims < 1 || numDims > MAX_DIMS) {- throw new IllegalArgumentException(- "numDims must be 1 .. " + MAX_DIMS + " (got: " + numDims + ")");- }- if (numIndexDims < 1 || numIndexDims > MAX_INDEX_DIMS) {- throw new IllegalArgumentException(- "numIndexDims must be 1 .. " + MAX_INDEX_DIMS + " (got: " + numIndexDims + ")");- }- if (numIndexDims > numDims) {- throw new IllegalArgumentException(- "numIndexDims cannot exceed numDims (" + numDims + ") (got: " + numIndexDims + ")");- }- if (maxPointsInLeafNode <= 0) {- throw new IllegalArgumentException(- "maxPointsInLeafNode must be > 0; got " + maxPointsInLeafNode);- }- if (maxPointsInLeafNode > ArrayUtil.MAX_ARRAY_LENGTH) {- throw new IllegalArgumentException(- "maxPointsInLeafNode must be <= ArrayUtil.MAX_ARRAY_LENGTH (= "- + ArrayUtil.MAX_ARRAY_LENGTH- + "); got "- + maxPointsInLeafNode);- }- if (maxMBSortInHeap < 0.0) {- throw new IllegalArgumentException(- "maxMBSortInHeap must be >= 0.0 (got: " + maxMBSortInHeap + ")");- }- if (totalPointCount < 0) {- throw new IllegalArgumentException(- "totalPointCount must be >=0 (got: " + totalPointCount + ")");- }- }-- private void initPointWriter() throws IOException {- assert pointWriter == null : "Point writer is already initialized";- // total point count is an estimation but the final point count must be equal or lower to that- // number.- if (totalPointCount > maxPointsSortInHeap) {- pointWriter = new OfflinePointWriter(config, tempDir, tempFileNamePrefix, "spill", 0);- tempInput = ((OfflinePointWriter) pointWriter).out;- } else {- pointWriter = new HeapPointWriter(config, Math.toIntExact(totalPointCount));- }- }-- public void add(byte[] packedValue, int docID) throws IOException {- if (packedValue.length != packedBytesLength) {- throw new IllegalArgumentException(- "packedValue should be length="- + packedBytesLength- + " (got: "- + packedValue.length- + ")");- }- if (pointCount >= totalPointCount) {- throw new IllegalStateException(- "totalPointCount="- + totalPointCount- + " was passed when we were created, but we just hit "- + (pointCount + 1)- + " values");- }- if (pointCount == 0) {- initPointWriter();- System.arraycopy(packedValue, 0, minPackedValue, 0, packedIndexBytesLength);- System.arraycopy(packedValue, 0, maxPackedValue, 0, packedIndexBytesLength);- } else {- for (int dim = 0; dim < numIndexDims; dim++) {- int offset = dim * bytesPerDim;- if (Arrays.compareUnsigned(- packedValue,- offset,- offset + bytesPerDim,- minPackedValue,- offset,- offset + bytesPerDim)- < 0) {- System.arraycopy(packedValue, offset, minPackedValue, offset, bytesPerDim);- }- if (Arrays.compareUnsigned(- packedValue,- offset,- offset + bytesPerDim,- maxPackedValue,- offset,- offset + bytesPerDim)- > 0) {- System.arraycopy(packedValue, offset, maxPackedValue, offset, bytesPerDim);- }- }- }- pointWriter.append(packedValue, docID);- pointCount++;- docsSeen.set(docID);- }-- /** How many points have been added so far */- public long getPointCount() {- return pointCount;- }-- private static class MergeReader {- private final PointValues.PointTree pointTree;- private final int packedBytesLength;- private final MergeState.DocMap docMap;- private final MergeIntersectsVisitor mergeIntersectsVisitor;-- /** Which doc in this block we are up to */- private int docBlockUpto;-- /** Current doc ID */- public int docID;-- /** Current packed value */- public final byte[] packedValue;-- public MergeReader(PointValues pointValues, MergeState.DocMap docMap) throws IOException {- this.packedBytesLength = pointValues.getBytesPerDimension() * pointValues.getNumDimensions();- this.pointTree = pointValues.getPointTree();- this.mergeIntersectsVisitor = new MergeIntersectsVisitor(packedBytesLength);- // move to first child of the tree and collect docs- while (pointTree.moveToChild()) {}- pointTree.visitDocValues(mergeIntersectsVisitor);- this.docMap = docMap;- this.packedValue = new byte[packedBytesLength];- }-- public boolean next() throws IOException {- // System.out.println("MR.next this=" + this);- while (true) {- if (docBlockUpto == mergeIntersectsVisitor.docsInBlock) {- if (collectNextLeaf() == false) {- assert mergeIntersectsVisitor.docsInBlock == 0;- return false;- }- assert mergeIntersectsVisitor.docsInBlock > 0;- docBlockUpto = 0;- }-- final int index = docBlockUpto++;- int oldDocID = mergeIntersectsVisitor.docIDs[index];-- int mappedDocID;- if (docMap == null) {- mappedDocID = oldDocID;- } else {- mappedDocID = docMap.get(oldDocID);- }-- if (mappedDocID != -1) {- // Not deleted!- docID = mappedDocID;- System.arraycopy(- mergeIntersectsVisitor.packedValues,- index * packedBytesLength,- packedValue,- 0,- packedBytesLength);- return true;- }- }- }-- private boolean collectNextLeaf() throws IOException {- assert pointTree.moveToChild() == false;- mergeIntersectsVisitor.reset();- do {- if (pointTree.moveToSibling()) {- // move to first child of this node and collect docs- while (pointTree.moveToChild()) {}- pointTree.visitDocValues(mergeIntersectsVisitor);- return true;- }- } while (pointTree.moveToParent());- return false;- }- }-- private static class MergeIntersectsVisitor implements IntersectVisitor {-- int docsInBlock = 0;- byte[] packedValues;- int[] docIDs;- private final int packedBytesLength;-- MergeIntersectsVisitor(int packedBytesLength) {- this.docIDs = new int[0];- this.packedValues = new byte[0];- this.packedBytesLength = packedBytesLength;- }-- void reset() {- docsInBlock = 0;- }-- @Override- public void grow(int count) {- assert docsInBlock == 0;- if (docIDs.length < count) {- docIDs = ArrayUtil.grow(docIDs, count);- int packedValuesSize = Math.toIntExact(docIDs.length * (long) packedBytesLength);- if (packedValuesSize > ArrayUtil.MAX_ARRAY_LENGTH) {- throw new IllegalStateException(- "array length must be <= to "- + ArrayUtil.MAX_ARRAY_LENGTH- + " but was: "- + packedValuesSize);- }- packedValues = ArrayUtil.growExact(packedValues, packedValuesSize);- }- }-- @Override- public void visit(int docID) {- throw new UnsupportedOperationException();- }-- @Override- public void visit(int docID, byte[] packedValue) {- System.arraycopy(- packedValue, 0, packedValues, docsInBlock * packedBytesLength, packedBytesLength);- docIDs[docsInBlock++] = docID;- }-- @Override- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {- return Relation.CELL_CROSSES_QUERY;- }- }-- private static Comparator<MergeReader> mergeComparator(int bytesPerDim) {- return ((Comparator<MergeReader>)- (a, b) -> {- assert a != b;- return Arrays.compareUnsigned(- a.packedValue, 0, bytesPerDim, b.packedValue, 0, bytesPerDim);- })- .thenComparingInt(mr -> mr.docID);- }-- /**- * Write a field from a {@link MutablePointTree}. This way of writing points is faster than- * regular writes with {@link BKDWriter60#add} since there is opportunity for reordering points- * before writing them to disk. This method does not use transient disk in order to reorder- * points.- */- public long writeField(IndexOutput out, String fieldName, MutablePointTree reader)- throws IOException {- if (numDataDims == 1) {- return writeField1Dim(out, fieldName, reader);- } else {- return writeFieldNDims(out, fieldName, reader);- }- }-- private void computePackedValueBounds(- MutablePointTree values,- int from,- int to,- byte[] minPackedValue,- byte[] maxPackedValue,- BytesRef scratch) {- if (from == to) {- return;- }- values.getValue(from, scratch);- System.arraycopy(scratch.bytes, scratch.offset, minPackedValue, 0, packedIndexBytesLength);- System.arraycopy(scratch.bytes, scratch.offset, maxPackedValue, 0, packedIndexBytesLength);- for (int i = from + 1; i < to; ++i) {- values.getValue(i, scratch);- for (int dim = 0; dim < numIndexDims; dim++) {- final int startOffset = dim * bytesPerDim;- final int endOffset = startOffset + bytesPerDim;- if (Arrays.compareUnsigned(- scratch.bytes,- scratch.offset + startOffset,- scratch.offset + endOffset,- minPackedValue,- startOffset,- endOffset)- < 0) {- System.arraycopy(- scratch.bytes,- scratch.offset + startOffset,- minPackedValue,- startOffset,- bytesPerDim);- } else if (Arrays.compareUnsigned(- scratch.bytes,- scratch.offset + startOffset,- scratch.offset + endOffset,- maxPackedValue,- startOffset,- endOffset)- > 0) {- System.arraycopy(- scratch.bytes,- scratch.offset + startOffset,- maxPackedValue,- startOffset,- bytesPerDim);- }- }- }- }-- /* In the 2+D case, we recursively pick the split dimension, compute the- * median value and partition other values around it. */- private long writeFieldNDims(IndexOutput out, String fieldName, MutablePointTree values)- throws IOException {- if (pointCount != 0) {- throw new IllegalStateException("cannot mix add and writeField");- }-- // Catch user silliness:- if (finished == true) {- throw new IllegalStateException("already finished");- }-- // Mark that we already finished:- finished = true;-- long countPerLeaf = pointCount = values.size();- long innerNodeCount = 1;-- while (countPerLeaf > maxPointsInLeafNode) {- countPerLeaf = (countPerLeaf + 1) / 2;- innerNodeCount *= 2;- }-- int numLeaves = Math.toIntExact(innerNodeCount);-- checkMaxLeafNodeCount(numLeaves);-- final byte[] splitPackedValues = new byte[numLeaves * (bytesPerDim + 1)];- final long[] leafBlockFPs = new long[numLeaves];-- // compute the min/max for this slice- computePackedValueBounds(- values, 0, Math.toIntExact(pointCount), minPackedValue, maxPackedValue, scratchBytesRef1);- for (int i = 0; i < Math.toIntExact(pointCount); ++i) {- docsSeen.set(values.getDocID(i));- }-- final int[] parentSplits = new int[numIndexDims];- build(- 1,- numLeaves,- values,- 0,- Math.toIntExact(pointCount),- out,- minPackedValue.clone(),- maxPackedValue.clone(),- parentSplits,- splitPackedValues,- leafBlockFPs,- new int[maxPointsInLeafNode]);- assert Arrays.equals(parentSplits, new int[numIndexDims]);-- long indexFP = out.getFilePointer();- writeIndex(out, Math.toIntExact(countPerLeaf), leafBlockFPs, splitPackedValues);- return indexFP;- }-- /* In the 1D case, we can simply sort points in ascending order and use the- * same writing logic as we use at merge time. */- private long writeField1Dim(IndexOutput out, String fieldName, MutablePointTree reader)- throws IOException {- MutablePointTreeReaderUtils.sort(config, maxDoc, reader, 0, Math.toIntExact(reader.size()));-- final OneDimensionBKDWriter oneDimWriter = new OneDimensionBKDWriter(out);-- reader.visitDocValues(- new IntersectVisitor() {-- @Override- public void visit(int docID, byte[] packedValue) throws IOException {- oneDimWriter.add(packedValue, docID);- }-- @Override- public void visit(int docID) throws IOException {- throw new IllegalStateException();- }-- @Override- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {- return Relation.CELL_CROSSES_QUERY;- }- });-- return oneDimWriter.finish();- }-- /**- * More efficient bulk-add for incoming {@link BKDReader}s. This does a merge sort of the already- * sorted values and currently only works when numDims==1. This returns -1 if all documents- * containing dimensional values were deleted.- */- public long merge(IndexOutput out, List<MergeState.DocMap> docMaps, List<PointValues> readers)- throws IOException {- assert docMaps == null || readers.size() == docMaps.size();-- PriorityQueue<MergeReader> queue =- PriorityQueue.usingComparator(readers.size(), mergeComparator(config.bytesPerDim()));-- for (int i = 0; i < readers.size(); i++) {- PointValues pointValues = readers.get(i);- assert pointValues.getNumDimensions() == config.numDims()- && pointValues.getBytesPerDimension() == config.bytesPerDim()- && pointValues.getNumIndexDimensions() == config.numIndexDims();- MergeState.DocMap docMap;- if (docMaps == null) {- docMap = null;- } else {- docMap = docMaps.get(i);- }- MergeReader reader = new MergeReader(pointValues, docMap);- if (reader.next()) {- queue.add(reader);- }- }-- OneDimensionBKDWriter oneDimWriter = new OneDimensionBKDWriter(out);-- while (queue.size() != 0) {- MergeReader reader = queue.top();- // System.out.println("iter reader=" + reader);-- oneDimWriter.add(reader.packedValue, reader.docID);-- if (reader.next()) {- queue.updateTop();- } else {- // This segment was exhausted- queue.pop();- }- }-- return oneDimWriter.finish();- }-- private class OneDimensionBKDWriter {-- final IndexOutput out;- final List<Long> leafBlockFPs = new ArrayList<>();- final List<byte[]> leafBlockStartValues = new ArrayList<>();- final byte[] leafValues = new byte[maxPointsInLeafNode * packedBytesLength];- final int[] leafDocs = new int[maxPointsInLeafNode];- private long valueCount;- private int leafCount;- private int leafCardinality;-- OneDimensionBKDWriter(IndexOutput out) {- if (numIndexDims != 1) {- throw new UnsupportedOperationException("numIndexDims must be 1 but got " + numIndexDims);- }- if (pointCount != 0) {- throw new IllegalStateException("cannot mix add and merge");- }-- // Catch user silliness:- if (finished == true) {- throw new IllegalStateException("already finished");- }-- // Mark that we already finished:- finished = true;-- this.out = out;-- lastPackedValue = new byte[packedBytesLength];- }-- // for asserts- final byte[] lastPackedValue;- private int lastDocID;-- void add(byte[] packedValue, int docID) throws IOException {- assert valueInOrder(- valueCount + leafCount, 0, lastPackedValue, packedValue, 0, docID, lastDocID);-- if (leafCount == 0- || Arrays.mismatch(- leafValues,- (leafCount - 1) * bytesPerDim,- leafCount * bytesPerDim,- packedValue,- 0,- bytesPerDim)- != -1) {- leafCardinality++;- }- System.arraycopy(- packedValue, 0, leafValues, leafCount * packedBytesLength, packedBytesLength);- leafDocs[leafCount] = docID;- docsSeen.set(docID);- leafCount++;-- if (valueCount + leafCount > totalPointCount) {- throw new IllegalStateException(- "totalPointCount="- + totalPointCount- + " was passed when we were created, but we just hit "- + (valueCount + leafCount)- + " values");- }-- if (leafCount == maxPointsInLeafNode) {- // We write a block once we hit exactly the max count ... this is different from- // when we write N > 1 dimensional points where we write between max/2 and max per leaf- // block- writeLeafBlock(leafCardinality);- leafCardinality = 0;- leafCount = 0;- }-- assert (lastDocID = docID) >= 0; // only assign when asserts are enabled- }-- public long finish() throws IOException {- if (leafCount > 0) {- writeLeafBlock(leafCardinality);- leafCardinality = 0;- leafCount = 0;- }-- if (valueCount == 0) {- return -1;- }-- pointCount = valueCount;-- long indexFP = out.getFilePointer();-- int numInnerNodes = leafBlockStartValues.size();-- // System.out.println("BKDW: now rotate numInnerNodes=" + numInnerNodes + " leafBlockStarts="- // + leafBlockStartValues.size());-- byte[] index = new byte[(1 + numInnerNodes) * (1 + bytesPerDim)];- rotateToTree(1, 0, numInnerNodes, index, leafBlockStartValues);- long[] arr = new long[leafBlockFPs.size()];- for (int i = 0; i < leafBlockFPs.size(); i++) {- arr[i] = leafBlockFPs.get(i);- }- writeIndex(out, maxPointsInLeafNode, arr, index);- return indexFP;- }-- private void writeLeafBlock(int leafCardinality) throws IOException {- assert leafCount != 0;- if (valueCount == 0) {- System.arraycopy(leafValues, 0, minPackedValue, 0, packedIndexBytesLength);- }- System.arraycopy(- leafValues,- (leafCount - 1) * packedBytesLength,- maxPackedValue,- 0,- packedIndexBytesLength);-- valueCount += leafCount;-- if (leafBlockFPs.size() > 0) {- // Save the first (minimum) value in each leaf block except the first, to build the split- // value index in the end:- leafBlockStartValues.add(ArrayUtil.copyOfSubArray(leafValues, 0, packedBytesLength));- }- leafBlockFPs.add(out.getFilePointer());- checkMaxLeafNodeCount(leafBlockFPs.size());-- // Find per-dim common prefix:- int offset = (leafCount - 1) * packedBytesLength;- int prefix =- Arrays.mismatch(leafValues, 0, bytesPerDim, leafValues, offset, offset + bytesPerDim);- if (prefix == -1) {- prefix = bytesPerDim;- }-- commonPrefixLengths[0] = prefix;-- writeLeafBlockDocs(out, leafDocs, 0, leafCount);- writeCommonPrefixes(out, commonPrefixLengths, leafValues);-- scratchBytesRef1.length = packedBytesLength;- scratchBytesRef1.bytes = leafValues;-- final IntFunction<BytesRef> packedValues =- new IntFunction<>() {- @Override- public BytesRef apply(int i) {- scratchBytesRef1.offset = packedBytesLength * i;- return scratchBytesRef1;- }- };- assert valuesInOrderAndBounds(- leafCount,- 0,- ArrayUtil.copyOfSubArray(leafValues, 0, packedBytesLength),- ArrayUtil.copyOfSubArray(- leafValues, (leafCount - 1) * packedBytesLength, leafCount * packedBytesLength),- packedValues,- leafDocs,- 0);- writeLeafBlockPackedValues(- out, commonPrefixLengths, leafCount, 0, packedValues, leafCardinality);- }- }-- // TODO: there must be a simpler way?- private void rotateToTree(- int nodeID, int offset, int count, byte[] index, List<byte[]> leafBlockStartValues) {- // System.out.println("ROTATE: nodeID=" + nodeID + " offset=" + offset + " count=" + count + "- // bpd=" + bytesPerDim + " index.length=" + index.length);- if (count == 1) {- // Leaf index node- // System.out.println(" leaf index node");- // System.out.println(" index[" + nodeID + "] = blockStartValues[" + offset + "]");- System.arraycopy(- leafBlockStartValues.get(offset), 0, index, nodeID * (1 + bytesPerDim) + 1, bytesPerDim);- } else if (count > 1) {- // Internal index node: binary partition of count- int countAtLevel = 1;- int totalCount = 0;- while (true) {- int countLeft = count - totalCount;- // System.out.println(" cycle countLeft=" + countLeft + " coutAtLevel=" + countAtLevel);- if (countLeft <= countAtLevel) {- // This is the last level, possibly partially filled:- int lastLeftCount = Math.min(countAtLevel / 2, countLeft);- assert lastLeftCount >= 0;- int leftHalf = (totalCount - 1) / 2 + lastLeftCount;-- int rootOffset = offset + leftHalf;- /*- System.out.println(" last left count " + lastLeftCount);- System.out.println(" leftHalf " + leftHalf + " rightHalf=" + (count-leftHalf-1));- System.out.println(" rootOffset=" + rootOffset);- */-- System.arraycopy(- leafBlockStartValues.get(rootOffset),- 0,- index,- nodeID * (1 + bytesPerDim) + 1,- bytesPerDim);- // System.out.println(" index[" + nodeID + "] = blockStartValues[" + rootOffset + "]");-- // TODO: we could optimize/specialize, when we know it's simply fully balanced binary tree- // under here, to save this while loop on each recursion-- // Recurse left- rotateToTree(2 * nodeID, offset, leftHalf, index, leafBlockStartValues);-- // Recurse right- rotateToTree(- 2 * nodeID + 1, rootOffset + 1, count - leftHalf - 1, index, leafBlockStartValues);- return;- }- totalCount += countAtLevel;- countAtLevel *= 2;- }- } else {- assert count == 0;- }- }-- // TODO: if we fixed each partition step to just record the file offset at the "split point", we- // could probably handle variable length- // encoding and not have our own ByteSequencesReader/Writer-- // useful for debugging:- /*- private void printPathSlice(String desc, PathSlice slice, int dim) throws IOException {- System.out.println(" " + desc + " dim=" + dim + " count=" + slice.count + ":");- try(PointReader r = slice.writer.getReader(slice.start, slice.count)) {- int count = 0;- while (r.next()) {- byte[] v = r.packedValue();- System.out.println(" " + count + ": " + new BytesRef(v, dim*bytesPerDim, bytesPerDim));- count++;- if (count == slice.count) {- break;- }- }- }- }- */-- private void checkMaxLeafNodeCount(int numLeaves) {- if ((1 + bytesPerDim) * (long) numLeaves > ArrayUtil.MAX_ARRAY_LENGTH) {- throw new IllegalStateException(- "too many nodes; increase maxPointsInLeafNode (currently "- + maxPointsInLeafNode- + ") and reindex");- }- }-- /**- * Writes the BKD tree to the provided {@link IndexOutput} and returns the file offset where index- * was written.- */- public long finish(IndexOutput out) throws IOException {- // System.out.println("\nBKDTreeWriter.finish pointCount=" + pointCount + " out=" + out + "- // heapWriter=" + heapPointWriter);-- // TODO: specialize the 1D case? it's much faster at indexing time (no partitioning on- // recurse...)-- // Catch user silliness:- if (finished == true) {- throw new IllegalStateException("already finished");- }-- if (pointCount == 0) {- throw new IllegalStateException("must index at least one point");- }-- // mark as finished- finished = true;-- pointWriter.close();- BKDRadixSelector.PathSlice points = new BKDRadixSelector.PathSlice(pointWriter, 0, pointCount);- // clean up pointers- tempInput = null;- pointWriter = null;-- long countPerLeaf = pointCount;- long innerNodeCount = 1;-- while (countPerLeaf > maxPointsInLeafNode) {- countPerLeaf = (countPerLeaf + 1) / 2;- innerNodeCount *= 2;- }-- int numLeaves = (int) innerNodeCount;-- checkMaxLeafNodeCount(numLeaves);-- // NOTE: we could save the 1+ here, to use a bit less heap at search time, but then we'd need a- // somewhat costly check at each- // step of the recursion to recompute the split dim:-- // Indexed by nodeID, but first (root) nodeID is 1. We do 1+ because the lead byte at each- // recursion says which dim we split on.- byte[] splitPackedValues = new byte[Math.toIntExact(numLeaves * (1 + bytesPerDim))];-- // +1 because leaf count is power of 2 (e.g. 8), and innerNodeCount is power of 2 minus 1 (e.g.- // 7)- long[] leafBlockFPs = new long[numLeaves];-- // Make sure the math above "worked":- assert pointCount / numLeaves <= maxPointsInLeafNode- : "pointCount="- + pointCount- + " numLeaves="- + numLeaves- + " maxPointsInLeafNode="- + maxPointsInLeafNode;-- // We re-use the selector so we do not need to create an object every time.- BKDRadixSelector radixSelector =- new BKDRadixSelector(config, maxPointsSortInHeap, tempDir, tempFileNamePrefix);-- boolean success = false;- try {-- final int[] parentSplits = new int[numIndexDims];- build(- 1,- numLeaves,- points,- out,- radixSelector,- minPackedValue.clone(),- maxPackedValue.clone(),- parentSplits,- splitPackedValues,- leafBlockFPs,- new int[maxPointsInLeafNode]);- assert Arrays.equals(parentSplits, new int[numIndexDims]);-- // If no exception, we should have cleaned everything up:- assert tempDir.getCreatedFiles().isEmpty();- // System.out.println("write time: " + ((System.nanoTime() - t1) / (double)- // TimeUnit.MILLISECONDS.toNanos(1)) + " ms");-- success = true;- } finally {- if (success == false) {- IOUtils.deleteFilesIgnoringExceptions(tempDir, tempDir.getCreatedFiles());- }- }-- // System.out.println("Total nodes: " + innerNodeCount);-- // Write index:- long indexFP = out.getFilePointer();- writeIndex(out, Math.toIntExact(countPerLeaf), leafBlockFPs, splitPackedValues);- return indexFP;- }-- /** Packs the two arrays, representing a balanced binary tree, into a compact byte[] structure. */- private byte[] packIndex(long[] leafBlockFPs, byte[] splitPackedValues) throws IOException {-- int numLeaves = leafBlockFPs.length;-- // Possibly rotate the leaf block FPs, if the index not fully balanced binary tree (only happens- // if it was created by OneDimensionBKDWriter). In this case the leaf nodes may straddle the- // two bottom- // levels of the binary tree:- if (numIndexDims == 1 && numLeaves > 1) {- int levelCount = 2;- while (true) {- if (numLeaves >= levelCount && numLeaves <= 2 * levelCount) {- int lastLevel = 2 * (numLeaves - levelCount);- assert lastLevel >= 0;- if (lastLevel != 0) {- // Last level is partially filled, so we must rotate the leaf FPs to match. We do this- // here, after loading- // at read-time, so that we can still delta code them on disk at write:- long[] newLeafBlockFPs = new long[numLeaves];- System.arraycopy(- leafBlockFPs, lastLevel, newLeafBlockFPs, 0, leafBlockFPs.length - lastLevel);- System.arraycopy(- leafBlockFPs, 0, newLeafBlockFPs, leafBlockFPs.length - lastLevel, lastLevel);- leafBlockFPs = newLeafBlockFPs;- }- break;- }-- levelCount *= 2;- }- }-- /* Reused while packing the index */- ByteBuffersDataOutput writeBuffer = new ByteBuffersDataOutput();-- // This is the "file" we append the byte[] to:- List<byte[]> blocks = new ArrayList<>();- byte[] lastSplitValues = new byte[bytesPerDim * numIndexDims];- // System.out.println("\npack index");- int totalSize =- recursePackIndex(- writeBuffer,- leafBlockFPs,- splitPackedValues,- 0l,- blocks,- 1,- lastSplitValues,- new boolean[numIndexDims],- false);-- // Compact the byte[] blocks into single byte index:- byte[] index = new byte[totalSize];- int upto = 0;- for (byte[] block : blocks) {- System.arraycopy(block, 0, index, upto, block.length);- upto += block.length;- }- assert upto == totalSize;-- return index;- }-- /** Appends the current contents of writeBuffer as another block on the growing in-memory file */- private int appendBlock(ByteBuffersDataOutput writeBuffer, List<byte[]> blocks)- throws IOException {- byte[] block = writeBuffer.toArrayCopy();- blocks.add(block);- writeBuffer.reset();- return block.length;- }-- /**- * lastSplitValues is per-dimension split value previously seen; we use this to prefix-code the- * split byte[] on each inner node- */- private int recursePackIndex(- ByteBuffersDataOutput writeBuffer,- long[] leafBlockFPs,- byte[] splitPackedValues,- long minBlockFP,- List<byte[]> blocks,- int nodeID,- byte[] lastSplitValues,- boolean[] negativeDeltas,- boolean isLeft)- throws IOException {- if (nodeID >= leafBlockFPs.length) {- int leafID = nodeID - leafBlockFPs.length;- // System.out.println("recursePack leaf nodeID=" + nodeID);-- // In the unbalanced case it's possible the left most node only has one child:- if (leafID < leafBlockFPs.length) {- long delta = leafBlockFPs[leafID] - minBlockFP;- if (isLeft) {- assert delta == 0;- return 0;- } else {- assert nodeID == 1 || delta > 0 : "nodeID=" + nodeID;- writeBuffer.writeVLong(delta);- return appendBlock(writeBuffer, blocks);- }- } else {- return 0;- }- } else {- long leftBlockFP;- if (isLeft == false) {- leftBlockFP = getLeftMostLeafBlockFP(leafBlockFPs, nodeID);- long delta = leftBlockFP - minBlockFP;- assert nodeID == 1 || delta > 0- : "expected nodeID=1 or delta > 0; got nodeID=" + nodeID + " and delta=" + delta;- writeBuffer.writeVLong(delta);- } else {- // The left tree's left most leaf block FP is always the minimal FP:- leftBlockFP = minBlockFP;- }-- int address = nodeID * (1 + bytesPerDim);- int splitDim = splitPackedValues[address++] & 0xff;-- // System.out.println("recursePack inner nodeID=" + nodeID + " splitDim=" + splitDim + "- // splitValue=" + new BytesRef(splitPackedValues, address, bytesPerDim));-- // find common prefix with last split value in this dim:- int prefix =- Arrays.mismatch(- splitPackedValues,- address,- address + bytesPerDim,- lastSplitValues,- splitDim * bytesPerDim,- splitDim * bytesPerDim + bytesPerDim);- if (prefix == -1) {- prefix = bytesPerDim;- }-- // System.out.println("writeNodeData nodeID=" + nodeID + " splitDim=" + splitDim + " numDims="- // + numDims + " bytesPerDim=" + bytesPerDim + " prefix=" + prefix);-- int firstDiffByteDelta;- if (prefix < bytesPerDim) {- // System.out.println(" delta byte cur=" +- // Integer.toHexString(splitPackedValues[address+prefix]&0xFF) + " prev=" +- // Integer.toHexString(lastSplitValues[splitDim * bytesPerDim + prefix]&0xFF) + " negated?="- // + negativeDeltas[splitDim]);- firstDiffByteDelta =- (splitPackedValues[address + prefix] & 0xFF)- - (lastSplitValues[splitDim * bytesPerDim + prefix] & 0xFF);- if (negativeDeltas[splitDim]) {- firstDiffByteDelta = -firstDiffByteDelta;- }- // System.out.println(" delta=" + firstDiffByteDelta);- assert firstDiffByteDelta > 0;- } else {- firstDiffByteDelta = 0;- }-- // pack the prefix, splitDim and delta first diff byte into a single vInt:- int code = (firstDiffByteDelta * (1 + bytesPerDim) + prefix) * numIndexDims + splitDim;-- // System.out.println(" code=" + code);- // System.out.println(" splitValue=" + new BytesRef(splitPackedValues, address,- // bytesPerDim));-- writeBuffer.writeVInt(code);-- // write the split value, prefix coded vs. our parent's split value:- int suffix = bytesPerDim - prefix;- byte[] savSplitValue = new byte[suffix];- if (suffix > 1) {- writeBuffer.writeBytes(splitPackedValues, address + prefix + 1, suffix - 1);- }-- byte[] cmp = lastSplitValues.clone();-- System.arraycopy(lastSplitValues, splitDim * bytesPerDim + prefix, savSplitValue, 0, suffix);-- // copy our split value into lastSplitValues for our children to prefix-code against- System.arraycopy(- splitPackedValues,- address + prefix,- lastSplitValues,- splitDim * bytesPerDim + prefix,- suffix);-- int numBytes = appendBlock(writeBuffer, blocks);-- // placeholder for left-tree numBytes; we need this so that at search time if we only need to- // recurse into the right sub-tree we can- // quickly seek to its starting point- int idxSav = blocks.size();- blocks.add(null);-- boolean savNegativeDelta = negativeDeltas[splitDim];- negativeDeltas[splitDim] = true;-- int leftNumBytes =- recursePackIndex(- writeBuffer,- leafBlockFPs,- splitPackedValues,- leftBlockFP,- blocks,- 2 * nodeID,- lastSplitValues,- negativeDeltas,- true);-- if (nodeID * 2 < leafBlockFPs.length) {- writeBuffer.writeVInt(leftNumBytes);- } else {- assert leftNumBytes == 0 : "leftNumBytes=" + leftNumBytes;- }-- byte[] bytes2 = writeBuffer.toArrayCopy();- writeBuffer.reset();- int numBytes2 = bytes2.length;- writeBuffer.reset();- // replace our placeholder:- blocks.set(idxSav, bytes2);-- negativeDeltas[splitDim] = false;- int rightNumBytes =- recursePackIndex(- writeBuffer,- leafBlockFPs,- splitPackedValues,- leftBlockFP,- blocks,- 2 * nodeID + 1,- lastSplitValues,- negativeDeltas,- false);-- negativeDeltas[splitDim] = savNegativeDelta;-- // restore lastSplitValues to what caller originally passed us:- System.arraycopy(savSplitValue, 0, lastSplitValues, splitDim * bytesPerDim + prefix, suffix);-- assert Arrays.equals(lastSplitValues, cmp);-- return numBytes + numBytes2 + leftNumBytes + rightNumBytes;- }- }-- private long getLeftMostLeafBlockFP(long[] leafBlockFPs, int nodeID) {- // TODO: can we do this cheaper, e.g. a closed form solution instead of while loop? Or- // change the recursion while packing the index to return this left-most leaf block FP- // from each recursion instead?- //- // Still, the overall cost here is minor: this method's cost is O(log(N)), and while writing- // we call it O(N) times (N = number of leaf blocks)- while (nodeID < leafBlockFPs.length) {- nodeID *= 2;- }- int leafID = nodeID - leafBlockFPs.length;- long result = leafBlockFPs[leafID];- if (result < 0) {- throw new AssertionError(result + " for leaf " + leafID);- }- return result;- }-- private void writeIndex(- IndexOutput out, int countPerLeaf, long[] leafBlockFPs, byte[] splitPackedValues)- throws IOException {- byte[] packedIndex = packIndex(leafBlockFPs, splitPackedValues);- writeIndex(out, countPerLeaf, leafBlockFPs.length, packedIndex);- }-- private void writeIndex(IndexOutput out, int countPerLeaf, int numLeaves, byte[] packedIndex)- throws IOException {-- CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT);- out.writeVInt(numDataDims);- out.writeVInt(numIndexDims);- out.writeVInt(countPerLeaf);- out.writeVInt(bytesPerDim);-- assert numLeaves > 0;- out.writeVInt(numLeaves);- out.writeBytes(minPackedValue, 0, packedIndexBytesLength);- out.writeBytes(maxPackedValue, 0, packedIndexBytesLength);-- out.writeVLong(pointCount);- out.writeVInt(docsSeen.cardinality());- out.writeVInt(packedIndex.length);- out.writeBytes(packedIndex, 0, packedIndex.length);- }-- private void writeLeafBlockDocs(DataOutput out, int[] docIDs, int start, int count)- throws IOException {- assert count > 0 : "maxPointsInLeafNode=" + maxPointsInLeafNode;- out.writeVInt(count);- DocIdsWriter.writeDocIds(docIDs, start, count, out);- }-- private void writeLeafBlockPackedValues(- DataOutput out,- int[] commonPrefixLengths,- int count,- int sortedDim,- IntFunction<BytesRef> packedValues,- int leafCardinality)- throws IOException {- int prefixLenSum = Arrays.stream(commonPrefixLengths).sum();- if (prefixLenSum == packedBytesLength) {- // all values in this block are equal- out.writeByte((byte) -1);- } else {- assert commonPrefixLengths[sortedDim] < bytesPerDim;- // estimate if storing the values with cardinality is cheaper than storing all values.- int compressedByteOffset = sortedDim * bytesPerDim + commonPrefixLengths[sortedDim];- int highCardinalityCost;- int lowCardinalityCost;- if (count == leafCardinality) {- // all values in this block are different- highCardinalityCost = 0;- lowCardinalityCost = 1;- } else {- // compute cost of runLen compression- int numRunLens = 0;- for (int i = 0; i < count; ) {- // do run-length compression on the byte at compressedByteOffset- int runLen = runLen(packedValues, i, Math.min(i + 0xff, count), compressedByteOffset);- assert runLen <= 0xff;- numRunLens++;- i += runLen;- }- // Add cost of runLen compression- highCardinalityCost = count * (packedBytesLength - prefixLenSum - 1) + 2 * numRunLens;- // +1 is the byte needed for storing the cardinality- lowCardinalityCost = leafCardinality * (packedBytesLength - prefixLenSum + 1);- }- if (lowCardinalityCost <= highCardinalityCost) {- out.writeByte((byte) -2);- writeLowCardinalityLeafBlockPackedValues(out, commonPrefixLengths, count, packedValues);- } else {- out.writeByte((byte) sortedDim);- writeHighCardinalityLeafBlockPackedValues(- out, commonPrefixLengths, count, sortedDim, packedValues, compressedByteOffset);- }- }- }-- private void writeLowCardinalityLeafBlockPackedValues(- DataOutput out, int[] commonPrefixLengths, int count, IntFunction<BytesRef> packedValues)- throws IOException {- if (numIndexDims != 1) {- writeActualBounds(out, commonPrefixLengths, count, packedValues);- }- BytesRef value = packedValues.apply(0);- System.arraycopy(value.bytes, value.offset, scratch1, 0, packedBytesLength);- int cardinality = 1;- for (int i = 1; i < count; i++) {- value = packedValues.apply(i);- for (int dim = 0; dim < numDataDims; dim++) {- final int start = dim * bytesPerDim + commonPrefixLengths[dim];- final int end = dim * bytesPerDim + bytesPerDim;- if (Arrays.mismatch(- value.bytes, value.offset + start, value.offset + end, scratch1, start, end)- != -1) {- out.writeVInt(cardinality);- for (int j = 0; j < numDataDims; j++) {- out.writeBytes(- scratch1,- j * bytesPerDim + commonPrefixLengths[j],- bytesPerDim - commonPrefixLengths[j]);- }- System.arraycopy(value.bytes, value.offset, scratch1, 0, packedBytesLength);- cardinality = 1;- break;- } else if (dim == numDataDims - 1) {- cardinality++;- }- }- }- out.writeVInt(cardinality);- for (int i = 0; i < numDataDims; i++) {- out.writeBytes(- scratch1, i * bytesPerDim + commonPrefixLengths[i], bytesPerDim - commonPrefixLengths[i]);- }- }-- private void writeHighCardinalityLeafBlockPackedValues(- DataOutput out,- int[] commonPrefixLengths,- int count,- int sortedDim,- IntFunction<BytesRef> packedValues,- int compressedByteOffset)- throws IOException {- if (numIndexDims != 1) {- writeActualBounds(out, commonPrefixLengths, count, packedValues);- }- commonPrefixLengths[sortedDim]++;- for (int i = 0; i < count; ) {- // do run-length compression on the byte at compressedByteOffset- int runLen = runLen(packedValues, i, Math.min(i + 0xff, count), compressedByteOffset);- assert runLen <= 0xff;- BytesRef first = packedValues.apply(i);- byte prefixByte = first.bytes[first.offset + compressedByteOffset];- out.writeByte(prefixByte);- out.writeByte((byte) runLen);- writeLeafBlockPackedValuesRange(out, commonPrefixLengths, i, i + runLen, packedValues);- i += runLen;- assert i <= count;- }- }-- private void writeActualBounds(- DataOutput out, int[] commonPrefixLengths, int count, IntFunction<BytesRef> packedValues)- throws IOException {- for (int dim = 0; dim < numIndexDims; ++dim) {- int commonPrefixLength = commonPrefixLengths[dim];- int suffixLength = bytesPerDim - commonPrefixLength;- if (suffixLength > 0) {- BytesRef[] minMax =- computeMinMax(- count, packedValues, dim * bytesPerDim + commonPrefixLength, suffixLength);- BytesRef min = minMax[0];- BytesRef max = minMax[1];- out.writeBytes(min.bytes, min.offset, min.length);- out.writeBytes(max.bytes, max.offset, max.length);- }- }- }-- /**- * Return an array that contains the min and max values for the [offset, offset+length] interval- * of the given {@link BytesRef}s.- */- private static BytesRef[] computeMinMax(- int count, IntFunction<BytesRef> packedValues, int offset, int length) {- assert length > 0;- BytesRefBuilder min = new BytesRefBuilder();- BytesRefBuilder max = new BytesRefBuilder();- BytesRef first = packedValues.apply(0);- min.copyBytes(first.bytes, first.offset + offset, length);- max.copyBytes(first.bytes, first.offset + offset, length);- for (int i = 1; i < count; ++i) {- BytesRef candidate = packedValues.apply(i);- if (Arrays.compareUnsigned(- min.bytes(),- 0,- length,- candidate.bytes,- candidate.offset + offset,- candidate.offset + offset + length)- > 0) {- min.copyBytes(candidate.bytes, candidate.offset + offset, length);- } else if (Arrays.compareUnsigned(- max.bytes(),- 0,- length,- candidate.bytes,- candidate.offset + offset,- candidate.offset + offset + length)- < 0) {- max.copyBytes(candidate.bytes, candidate.offset + offset, length);- }- }- return new BytesRef[] {min.get(), max.get()};- }-- private void writeLeafBlockPackedValuesRange(- DataOutput out,- int[] commonPrefixLengths,- int start,- int end,- IntFunction<BytesRef> packedValues)- throws IOException {- for (int i = start; i < end; ++i) {- BytesRef ref = packedValues.apply(i);- assert ref.length == packedBytesLength;-- for (int dim = 0; dim < numDataDims; dim++) {- int prefix = commonPrefixLengths[dim];- out.writeBytes(ref.bytes, ref.offset + dim * bytesPerDim + prefix, bytesPerDim - prefix);- }- }- }-- private static int runLen(- IntFunction<BytesRef> packedValues, int start, int end, int byteOffset) {- BytesRef first = packedValues.apply(start);- byte b = first.bytes[first.offset + byteOffset];- for (int i = start + 1; i < end; ++i) {- BytesRef ref = packedValues.apply(i);- byte b2 = ref.bytes[ref.offset + byteOffset];- assert Byte.toUnsignedInt(b2) >= Byte.toUnsignedInt(b);- if (b != b2) {- return i - start;- }- }- return end - start;- }-- private void writeCommonPrefixes(DataOutput out, int[] commonPrefixes, byte[] packedValue)- throws IOException {- for (int dim = 0; dim < numDataDims; dim++) {- out.writeVInt(commonPrefixes[dim]);- // System.out.println(commonPrefixes[dim] + " of " + bytesPerDim);- out.writeBytes(packedValue, dim * bytesPerDim, commonPrefixes[dim]);- }- }-- @Override- public void close() throws IOException {- finished = true;- if (tempInput != null) {- // NOTE: this should only happen on exception, e.g. caller calls close w/o calling finish:- try {- tempInput.close();- } finally {- tempDir.deleteFile(tempInput.getName());- tempInput = null;- }- }- }-- /**- * Called on exception, to check whether the checksum is also corrupt in this source, and add that- * information (checksum matched or didn't) as a suppressed exception.- */- private Error verifyChecksum(Throwable priorException, PointWriter writer) throws IOException {- assert priorException != null;-- // TODO: we could improve this, to always validate checksum as we recurse, if we shared left and- // right reader after recursing to children, and possibly within recursed children,- // since all together they make a single pass through the file. But this is a sizable re-org,- // and would mean leaving readers (IndexInputs) open for longer:- if (writer instanceof OfflinePointWriter) {- // We are reading from a temp file; go verify the checksum:- String tempFileName = ((OfflinePointWriter) writer).name;- if (tempDir.getCreatedFiles().contains(tempFileName)) {- try (ChecksumIndexInput in = tempDir.openChecksumInput(tempFileName)) {- CodecUtil.checkFooter(in, priorException);- }- }- }-- // We are reading from heap; nothing to add:- throw IOUtils.rethrowAlways(priorException);- }-- /** Called only in assert */- private boolean valueInBounds(- BytesRef packedValue, byte[] minPackedValue, byte[] maxPackedValue) {- for (int dim = 0; dim < numIndexDims; dim++) {- int offset = bytesPerDim * dim;- if (Arrays.compareUnsigned(- packedValue.bytes,- packedValue.offset + offset,- packedValue.offset + offset + bytesPerDim,- minPackedValue,- offset,- offset + bytesPerDim)- < 0) {- return false;- }- if (Arrays.compareUnsigned(- packedValue.bytes,- packedValue.offset + offset,- packedValue.offset + offset + bytesPerDim,- maxPackedValue,- offset,- offset + bytesPerDim)- > 0) {- return false;- }- }-- return true;- }-- /**- * Pick the next dimension to split.- *- * @param minPackedValue the min values for all dimensions- * @param maxPackedValue the max values for all dimensions- * @param parentSplits how many times each dim has been split on the parent levels- * @return the dimension to split- */- protected int split(byte[] minPackedValue, byte[] maxPackedValue, int[] parentSplits) {- // First look at whether there is a dimension that has split less than 2x less than- // the dim that has most splits, and return it if there is such a dimension and it- // does not only have equals values. This helps ensure all dimensions are indexed.- int maxNumSplits = 0;- for (int numSplits : parentSplits) {- maxNumSplits = Math.max(maxNumSplits, numSplits);- }- for (int dim = 0; dim < numIndexDims; ++dim) {- final int offset = dim * bytesPerDim;- if (parentSplits[dim] < maxNumSplits / 2- && Arrays.compareUnsigned(- minPackedValue,- offset,- offset + bytesPerDim,- maxPackedValue,- offset,- offset + bytesPerDim)- != 0) {- return dim;- }- }-- // Find which dim has the largest span so we can split on it:- int splitDim = -1;- for (int dim = 0; dim < numIndexDims; dim++) {- NumericUtils.subtract(bytesPerDim, dim, maxPackedValue, minPackedValue, scratchDiff);- if (splitDim == -1- || Arrays.compareUnsigned(scratchDiff, 0, bytesPerDim, scratch1, 0, bytesPerDim) > 0) {- System.arraycopy(scratchDiff, 0, scratch1, 0, bytesPerDim);- splitDim = dim;- }- }-- // System.out.println("SPLIT: " + splitDim);- return splitDim;- }-- /** Pull a partition back into heap once the point count is low enough while recursing. */- private HeapPointWriter switchToHeap(PointWriter source) throws IOException {- int count = Math.toIntExact(source.count());- try (PointReader reader = source.getReader(0, source.count());- HeapPointWriter writer = new HeapPointWriter(config, count)) {- for (int i = 0; i < count; i++) {- boolean hasNext = reader.next();- assert hasNext;- writer.append(reader.pointValue());- }- source.destroy();- return writer;- } catch (Throwable t) {- throw verifyChecksum(t, source);- }- }-- /* Recursively reorders the provided reader and writes the bkd-tree on the fly; this method is used- * when we are writing a new segment directly from IndexWriter's indexing buffer (MutablePointsReader). */- private void build(- int nodeID,- int leafNodeOffset,- MutablePointTree reader,- int from,- int to,- IndexOutput out,- byte[] minPackedValue,- byte[] maxPackedValue,- int[] parentSplits,- byte[] splitPackedValues,- long[] leafBlockFPs,- int[] spareDocIds)- throws IOException {-- if (nodeID >= leafNodeOffset) {- // leaf node- final int count = to - from;- assert count <= maxPointsInLeafNode;-- // Compute common prefixes- Arrays.fill(commonPrefixLengths, bytesPerDim);- reader.getValue(from, scratchBytesRef1);- for (int i = from + 1; i < to; ++i) {- reader.getValue(i, scratchBytesRef2);- for (int dim = 0; dim < numDataDims; dim++) {- final int offset = dim * bytesPerDim;- int dimensionPrefixLength = commonPrefixLengths[dim];- commonPrefixLengths[dim] =- Arrays.mismatch(- scratchBytesRef1.bytes,- scratchBytesRef1.offset + offset,- scratchBytesRef1.offset + offset + dimensionPrefixLength,- scratchBytesRef2.bytes,- scratchBytesRef2.offset + offset,- scratchBytesRef2.offset + offset + dimensionPrefixLength);- if (commonPrefixLengths[dim] == -1) {- commonPrefixLengths[dim] = dimensionPrefixLength;- }- }- }-- // Find the dimension that has the least number of unique bytes at commonPrefixLengths[dim]- FixedBitSet[] usedBytes = new FixedBitSet[numDataDims];- for (int dim = 0; dim < numDataDims; ++dim) {- if (commonPrefixLengths[dim] < bytesPerDim) {- usedBytes[dim] = new FixedBitSet(256);- }- }- for (int i = from + 1; i < to; ++i) {- for (int dim = 0; dim < numDataDims; dim++) {- if (usedBytes[dim] != null) {- byte b = reader.getByteAt(i, dim * bytesPerDim + commonPrefixLengths[dim]);- usedBytes[dim].set(Byte.toUnsignedInt(b));- }- }- }- int sortedDim = 0;- int sortedDimCardinality = Integer.MAX_VALUE;- for (int dim = 0; dim < numDataDims; ++dim) {- if (usedBytes[dim] != null) {- final int cardinality = usedBytes[dim].cardinality();- if (cardinality < sortedDimCardinality) {- sortedDim = dim;- sortedDimCardinality = cardinality;- }- }- }-- // sort by sortedDim- MutablePointTreeReaderUtils.sortByDim(- config,- sortedDim,- commonPrefixLengths,- reader,- from,- to,- scratchBytesRef1,- scratchBytesRef2);-- BytesRef comparator = scratchBytesRef1;- BytesRef collector = scratchBytesRef2;- reader.getValue(from, comparator);- int leafCardinality = 1;- for (int i = from + 1; i < to; ++i) {- reader.getValue(i, collector);- for (int dim = 0; dim < numDataDims; dim++) {- final int start = dim * bytesPerDim + commonPrefixLengths[dim];- final int end = dim * bytesPerDim + bytesPerDim;- if (Arrays.mismatch(- collector.bytes,- collector.offset + start,- collector.offset + end,- comparator.bytes,- comparator.offset + start,- comparator.offset + end)- != -1) {- leafCardinality++;- BytesRef scratch = collector;- collector = comparator;- comparator = scratch;- break;- }- }- }- // Save the block file pointer:- leafBlockFPs[nodeID - leafNodeOffset] = out.getFilePointer();-- // Write doc IDs- int[] docIDs = spareDocIds;- for (int i = from; i < to; ++i) {- docIDs[i - from] = reader.getDocID(i);- }- // System.out.println("writeLeafBlock pos=" + out.getFilePointer());- writeLeafBlockDocs(out, docIDs, 0, count);-- // Write the common prefixes:- reader.getValue(from, scratchBytesRef1);- System.arraycopy(- scratchBytesRef1.bytes, scratchBytesRef1.offset, scratch1, 0, packedBytesLength);- writeCommonPrefixes(out, commonPrefixLengths, scratch1);-- // Write the full values:- IntFunction<BytesRef> packedValues =- new IntFunction<>() {- @Override- public BytesRef apply(int i) {- reader.getValue(from + i, scratchBytesRef1);- return scratchBytesRef1;- }- };- assert valuesInOrderAndBounds(- count, sortedDim, minPackedValue, maxPackedValue, packedValues, docIDs, 0);- writeLeafBlockPackedValues(- out, commonPrefixLengths, count, sortedDim, packedValues, leafCardinality);-- } else {- // inner node-- final int splitDim;- // compute the split dimension and partition around it- if (numIndexDims == 1) {- splitDim = 0;- } else {- // for dimensions > 2 we recompute the bounds for the current inner node to help the- // algorithm choose best- // split dimensions. Because it is an expensive operation, the frequency we recompute the- // bounds is given- // by SPLITS_BEFORE_EXACT_BOUNDS.- if (nodeID > 1- && numIndexDims > 2- && Arrays.stream(parentSplits).sum() % SPLITS_BEFORE_EXACT_BOUNDS == 0) {- computePackedValueBounds(- reader, from, to, minPackedValue, maxPackedValue, scratchBytesRef1);- }- splitDim = split(minPackedValue, maxPackedValue, parentSplits);- }-- final int mid = (from + to + 1) >>> 1;-- int commonPrefixLen =- Arrays.mismatch(- minPackedValue,- splitDim * bytesPerDim,- splitDim * bytesPerDim + bytesPerDim,- maxPackedValue,- splitDim * bytesPerDim,- splitDim * bytesPerDim + bytesPerDim);- if (commonPrefixLen == -1) {- commonPrefixLen = bytesPerDim;- }-- MutablePointTreeReaderUtils.partition(- config,- maxDoc,- splitDim,- commonPrefixLen,- reader,- from,- to,- mid,- scratchBytesRef1,- scratchBytesRef2);-- // set the split value- final int address = nodeID * (1 + bytesPerDim);- splitPackedValues[address] = (byte) splitDim;- reader.getValue(mid, scratchBytesRef1);- System.arraycopy(- scratchBytesRef1.bytes,- scratchBytesRef1.offset + splitDim * bytesPerDim,- splitPackedValues,- address + 1,- bytesPerDim);-- byte[] minSplitPackedValue =- ArrayUtil.copyOfSubArray(minPackedValue, 0, packedIndexBytesLength);- byte[] maxSplitPackedValue =- ArrayUtil.copyOfSubArray(maxPackedValue, 0, packedIndexBytesLength);- System.arraycopy(- scratchBytesRef1.bytes,- scratchBytesRef1.offset + splitDim * bytesPerDim,- minSplitPackedValue,- splitDim * bytesPerDim,- bytesPerDim);- System.arraycopy(- scratchBytesRef1.bytes,- scratchBytesRef1.offset + splitDim * bytesPerDim,- maxSplitPackedValue,- splitDim * bytesPerDim,- bytesPerDim);-- // recurse- parentSplits[splitDim]++;- build(- nodeID * 2,- leafNodeOffset,- reader,- from,- mid,- out,- minPackedValue,- maxSplitPackedValue,- parentSplits,- splitPackedValues,- leafBlockFPs,- spareDocIds);- build(- nodeID * 2 + 1,- leafNodeOffset,- reader,- mid,- to,- out,- minSplitPackedValue,- maxPackedValue,- parentSplits,- splitPackedValues,- leafBlockFPs,- spareDocIds);- parentSplits[splitDim]--;- }- }-- private void computePackedValueBounds(- BKDRadixSelector.PathSlice slice, byte[] minPackedValue, byte[] maxPackedValue)- throws IOException {- try (PointReader reader = slice.writer().getReader(slice.start(), slice.count())) {- if (reader.next() == false) {- return;- }- BytesRef value = reader.pointValue().packedValue();- System.arraycopy(value.bytes, value.offset, minPackedValue, 0, packedIndexBytesLength);- System.arraycopy(value.bytes, value.offset, maxPackedValue, 0, packedIndexBytesLength);- while (reader.next()) {- value = reader.pointValue().packedValue();- for (int dim = 0; dim < numIndexDims; dim++) {- final int startOffset = dim * bytesPerDim;- final int endOffset = startOffset + bytesPerDim;- if (Arrays.compareUnsigned(- value.bytes,- value.offset + startOffset,- value.offset + endOffset,- minPackedValue,- startOffset,- endOffset)- < 0) {- System.arraycopy(- value.bytes, value.offset + startOffset, minPackedValue, startOffset, bytesPerDim);- } else if (Arrays.compareUnsigned(- value.bytes,- value.offset + startOffset,- value.offset + endOffset,- maxPackedValue,- startOffset,- endOffset)- > 0) {- System.arraycopy(- value.bytes, value.offset + startOffset, maxPackedValue, startOffset, bytesPerDim);- }- }- }- }- }-- /**- * The point writer contains the data that is going to be splitted using radix selection. /* This- * method is used when we are merging previously written segments, in the numDims > 1 case.- */- private void build(- int nodeID,- int leafNodeOffset,- BKDRadixSelector.PathSlice points,- IndexOutput out,- BKDRadixSelector radixSelector,- byte[] minPackedValue,- byte[] maxPackedValue,- int[] parentSplits,- byte[] splitPackedValues,- long[] leafBlockFPs,- int[] spareDocIds)- throws IOException {-- if (nodeID >= leafNodeOffset) {-- // Leaf node: write block- // We can write the block in any order so by default we write it sorted by the dimension that- // has the- // least number of unique bytes at commonPrefixLengths[dim], which makes compression more- // efficient- HeapPointWriter heapSource;- if (points.writer() instanceof HeapPointWriter == false) {- // Adversarial cases can cause this, e.g. merging big segments with most of the points- // deleted- heapSource = switchToHeap(points.writer());- } else {- heapSource = (HeapPointWriter) points.writer();- }-- int from = Math.toIntExact(points.start());- int to = Math.toIntExact(points.start() + points.count());- // we store common prefix on scratch1- computeCommonPrefixLength(heapSource, scratch1, from, to);-- int sortedDim = 0;- int sortedDimCardinality = Integer.MAX_VALUE;- FixedBitSet[] usedBytes = new FixedBitSet[numDataDims];- for (int dim = 0; dim < numDataDims; ++dim) {- if (commonPrefixLengths[dim] < bytesPerDim) {- usedBytes[dim] = new FixedBitSet(256);- }- }- // Find the dimension to compress- for (int dim = 0; dim < numDataDims; dim++) {- int prefix = commonPrefixLengths[dim];- if (prefix < bytesPerDim) {- int offset = dim * bytesPerDim;- for (int i = from; i < to; ++i) {- PointValue value = heapSource.getPackedValueSlice(i);- BytesRef packedValue = value.packedValue();- int bucket = packedValue.bytes[packedValue.offset + offset + prefix] & 0xff;- usedBytes[dim].set(bucket);- }- int cardinality = usedBytes[dim].cardinality();- if (cardinality < sortedDimCardinality) {- sortedDim = dim;- sortedDimCardinality = cardinality;- }- }- }-- // sort the chosen dimension- radixSelector.heapRadixSort(heapSource, from, to, sortedDim, commonPrefixLengths[sortedDim]);- // compute cardinality- int leafCardinality = heapSource.computeCardinality(from, to, commonPrefixLengths);-- // Save the block file pointer:- leafBlockFPs[nodeID - leafNodeOffset] = out.getFilePointer();- // System.out.println(" write leaf block @ fp=" + out.getFilePointer());-- // Write docIDs first, as their own chunk, so that at intersect time we can add all docIDs w/o- // loading the values:- int count = to - from;- assert count > 0 : "nodeID=" + nodeID + " leafNodeOffset=" + leafNodeOffset;- assert count <= spareDocIds.length : "count=" + count + " > length=" + spareDocIds.length;- // Write doc IDs- int[] docIDs = spareDocIds;- for (int i = 0; i < count; i++) {- docIDs[i] = heapSource.getPackedValueSlice(from + i).docID();- }- writeLeafBlockDocs(out, docIDs, 0, count);-- // TODO: minor opto: we don't really have to write the actual common prefixes, because- // BKDReader on recursing can regenerate it for us- // from the index, much like how terms dict does so from the FST:-- // Write the common prefixes:- writeCommonPrefixes(out, commonPrefixLengths, scratch1);-- // Write the full values:- IntFunction<BytesRef> packedValues =- new IntFunction<>() {- final BytesRef scratch = new BytesRef();-- {- scratch.length = packedBytesLength;- }-- @Override- public BytesRef apply(int i) {- PointValue value = heapSource.getPackedValueSlice(from + i);- return value.packedValue();- }- };- assert valuesInOrderAndBounds(- count, sortedDim, minPackedValue, maxPackedValue, packedValues, docIDs, 0);- writeLeafBlockPackedValues(- out, commonPrefixLengths, count, sortedDim, packedValues, leafCardinality);-- } else {- // Inner node: partition/recurse-- final int splitDim;- if (numIndexDims == 1) {- splitDim = 0;- } else {- // for dimensions > 2 we recompute the bounds for the current inner node to help the- // algorithm choose best- // split dimensions. Because it is an expensive operation, the frequency we recompute the- // bounds is given- // by SPLITS_BEFORE_EXACT_BOUNDS.- if (nodeID > 1- && numIndexDims > 2- && Arrays.stream(parentSplits).sum() % SPLITS_BEFORE_EXACT_BOUNDS == 0) {- computePackedValueBounds(points, minPackedValue, maxPackedValue);- }- splitDim = split(minPackedValue, maxPackedValue, parentSplits);- }-- assert nodeID < splitPackedValues.length- : "nodeID=" + nodeID + " splitValues.length=" + splitPackedValues.length;-- // How many points will be in the left tree:- long rightCount = points.count() / 2;- long leftCount = points.count() - rightCount;-- BKDRadixSelector.PathSlice[] slices = new BKDRadixSelector.PathSlice[2];-- int commonPrefixLen =- Arrays.mismatch(- minPackedValue,- splitDim * bytesPerDim,- splitDim * bytesPerDim + bytesPerDim,- maxPackedValue,- splitDim * bytesPerDim,- splitDim * bytesPerDim + bytesPerDim);- if (commonPrefixLen == -1) {- commonPrefixLen = bytesPerDim;- }-- byte[] splitValue =- radixSelector.select(- points,- slices,- points.start(),- points.start() + points.count(),- points.start() + leftCount,- splitDim,- commonPrefixLen);-- int address = nodeID * (1 + bytesPerDim);- splitPackedValues[address] = (byte) splitDim;- System.arraycopy(splitValue, 0, splitPackedValues, address + 1, bytesPerDim);-- byte[] minSplitPackedValue = new byte[packedIndexBytesLength];- System.arraycopy(minPackedValue, 0, minSplitPackedValue, 0, packedIndexBytesLength);-- byte[] maxSplitPackedValue = new byte[packedIndexBytesLength];- System.arraycopy(maxPackedValue, 0, maxSplitPackedValue, 0, packedIndexBytesLength);-- System.arraycopy(splitValue, 0, minSplitPackedValue, splitDim * bytesPerDim, bytesPerDim);- System.arraycopy(splitValue, 0, maxSplitPackedValue, splitDim * bytesPerDim, bytesPerDim);-- parentSplits[splitDim]++;- // Recurse on left tree:- build(- 2 * nodeID,- leafNodeOffset,- slices[0],- out,- radixSelector,- minPackedValue,- maxSplitPackedValue,- parentSplits,- splitPackedValues,- leafBlockFPs,- spareDocIds);-- // Recurse on right tree:- build(- 2 * nodeID + 1,- leafNodeOffset,- slices[1],- out,- radixSelector,- minSplitPackedValue,- maxPackedValue,- parentSplits,- splitPackedValues,- leafBlockFPs,- spareDocIds);-- parentSplits[splitDim]--;- }- }-- private void computeCommonPrefixLength(- HeapPointWriter heapPointWriter, byte[] commonPrefix, int from, int to) {- Arrays.fill(commonPrefixLengths, bytesPerDim);- PointValue value = heapPointWriter.getPackedValueSlice(from);- BytesRef packedValue = value.packedValue();- for (int dim = 0; dim < numDataDims; dim++) {- System.arraycopy(- packedValue.bytes,- packedValue.offset + dim * bytesPerDim,- commonPrefix,- dim * bytesPerDim,- bytesPerDim);- }- for (int i = from + 1; i < to; i++) {- value = heapPointWriter.getPackedValueSlice(i);- packedValue = value.packedValue();- for (int dim = 0; dim < numDataDims; dim++) {- if (commonPrefixLengths[dim] != 0) {- int j =- Arrays.mismatch(- commonPrefix,- dim * bytesPerDim,- dim * bytesPerDim + commonPrefixLengths[dim],- packedValue.bytes,- packedValue.offset + dim * bytesPerDim,- packedValue.offset + dim * bytesPerDim + commonPrefixLengths[dim]);- if (j != -1) {- commonPrefixLengths[dim] = j;- }- }- }- }- }-- // only called from assert- private boolean valuesInOrderAndBounds(- int count,- int sortedDim,- byte[] minPackedValue,- byte[] maxPackedValue,- IntFunction<BytesRef> values,- int[] docs,- int docsOffset)- throws IOException {- byte[] lastPackedValue = new byte[packedBytesLength];- int lastDoc = -1;- for (int i = 0; i < count; i++) {- BytesRef packedValue = values.apply(i);- assert packedValue.length == packedBytesLength;- assert valueInOrder(- i,- sortedDim,- lastPackedValue,- packedValue.bytes,- packedValue.offset,- docs[docsOffset + i],- lastDoc);- lastDoc = docs[docsOffset + i];-- // Make sure this value does in fact fall within this leaf cell:- assert valueInBounds(packedValue, minPackedValue, maxPackedValue);- }- return true;- }-- // only called from assert- private boolean valueInOrder(- long ord,- int sortedDim,- byte[] lastPackedValue,- byte[] packedValue,- int packedValueOffset,- int doc,- int lastDoc) {- int dimOffset = sortedDim * bytesPerDim;- if (ord > 0) {- int cmp =- Arrays.compareUnsigned(- lastPackedValue,- dimOffset,- dimOffset + bytesPerDim,- packedValue,- packedValueOffset + dimOffset,- packedValueOffset + dimOffset + bytesPerDim);- if (cmp > 0) {- throw new AssertionError(- "values out of order: last value="- + new BytesRef(lastPackedValue)- + " current value="- + new BytesRef(packedValue, packedValueOffset, packedBytesLength)- + " ord="- + ord);- }- if (cmp == 0 && numDataDims > numIndexDims) {- int dataOffset = numIndexDims * bytesPerDim;- cmp =- Arrays.compareUnsigned(- lastPackedValue,- dataOffset,- packedBytesLength,- packedValue,- packedValueOffset + dataOffset,- packedValueOffset + packedBytesLength);- if (cmp > 0) {- throw new AssertionError(- "data values out of order: last value="- + new BytesRef(lastPackedValue)- + " current value="- + new BytesRef(packedValue, packedValueOffset, packedBytesLength)- + " ord="- + ord);- }- }- if (cmp == 0 && doc < lastDoc) {- throw new AssertionError(- "docs out of order: last doc=" + lastDoc + " current doc=" + doc + " ord=" + ord);- }- }- System.arraycopy(packedValue, packedValueOffset, lastPackedValue, 0, packedBytesLength);- return true;- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/bkd/DocIdsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/bkd/DocIdsWriter.javadeleted file mode 100644index c460759cbe9..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/bkd/DocIdsWriter.java+++ /dev/null@@ -1,85 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene60.bkd;--import java.io.IOException;-import org.apache.lucene.store.DataOutput;--class DocIdsWriter {-- private DocIdsWriter() {}-- static void writeDocIds(int[] docIds, int start, int count, DataOutput out) throws IOException {- // docs can be sorted either when all docs in a block have the same value- // or when a segment is sorted- boolean sorted = true;- for (int i = 1; i < count; ++i) {- if (docIds[start + i - 1] > docIds[start + i]) {- sorted = false;- break;- }- }- if (sorted) {- out.writeByte((byte) 0);- int previous = 0;- for (int i = 0; i < count; ++i) {- int doc = docIds[start + i];- out.writeVInt(doc - previous);- previous = doc;- }- } else {- long max = 0;- for (int i = 0; i < count; ++i) {- max |= Integer.toUnsignedLong(docIds[start + i]);- }- if (max <= 0xffffff) {- out.writeByte((byte) 24);- // write them the same way we are reading them.- int i;- for (i = 0; i < count - 7; i += 8) {- int doc1 = docIds[start + i];- int doc2 = docIds[start + i + 1];- int doc3 = docIds[start + i + 2];- int doc4 = docIds[start + i + 3];- int doc5 = docIds[start + i + 4];- int doc6 = docIds[start + i + 5];- int doc7 = docIds[start + i + 6];- int doc8 = docIds[start + i + 7];- long l1 = (doc1 & 0xffffffL) << 40 | (doc2 & 0xffffffL) << 16 | ((doc3 >>> 8) & 0xffffL);- long l2 =- (doc3 & 0xffL) << 56- | (doc4 & 0xffffffL) << 32- | (doc5 & 0xffffffL) << 8- | ((doc6 >> 16) & 0xffL);- long l3 = (doc6 & 0xffffL) << 48 | (doc7 & 0xffffffL) << 24 | (doc8 & 0xffffffL);- out.writeLong(l1);- out.writeLong(l2);- out.writeLong(l3);- }- for (; i < count; ++i) {- out.writeShort((short) (docIds[start + i] >>> 8));- out.writeByte((byte) docIds[start + i]);- }- } else {- out.writeByte((byte) 32);- for (int i = 0; i < count; ++i) {- out.writeInt(docIds[start + i]);- }- }- }- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene70/Lucene70RWSegmentInfoFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene70/Lucene70RWSegmentInfoFormat.javadeleted file mode 100644index cf267f03c5a..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene70/Lucene70RWSegmentInfoFormat.java+++ /dev/null@@ -1,226 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--package org.apache.lucene.backward_codecs.lucene70;--import java.io.IOException;-import java.util.Set;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.SegmentInfo;-import org.apache.lucene.search.Sort;-import org.apache.lucene.search.SortField;-import org.apache.lucene.search.SortedNumericSelector;-import org.apache.lucene.search.SortedNumericSortField;-import org.apache.lucene.search.SortedSetSelector;-import org.apache.lucene.search.SortedSetSortField;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IndexOutput;-import org.apache.lucene.util.Version;--/** Writable version of Lucene70SegmentInfoFormat for testing */-public class Lucene70RWSegmentInfoFormat extends Lucene70SegmentInfoFormat {-- @Override- public void write(Directory dir, SegmentInfo si, IOContext ioContext) throws IOException {- final String fileName =- IndexFileNames.segmentFileName(si.name, "", Lucene70SegmentInfoFormat.SI_EXTENSION);-- try (IndexOutput output = EndiannessReverserUtil.createOutput(dir, fileName, ioContext)) {- // Only add the file once we've successfully created it, else IFD assert can trip:- si.addFile(fileName);- CodecUtil.writeIndexHeader(- output,- Lucene70SegmentInfoFormat.CODEC_NAME,- Lucene70SegmentInfoFormat.VERSION_CURRENT,- si.getId(),- "");-- writeSegmentInfo(output, si);-- CodecUtil.writeFooter(output);- }- }-- private void writeSegmentInfo(IndexOutput output, SegmentInfo si) throws IOException {- Version version = si.getVersion();- if (version.major < 7) {- throw new IllegalArgumentException(- "invalid major version: should be >= 7 but got: " + version.major + " segment=" + si);- }- // Write the Lucene version that created this segment, since 3.1- output.writeInt(version.major);- output.writeInt(version.minor);- output.writeInt(version.bugfix);-- // Write the min Lucene version that contributed docs to the segment, since 7.0- if (si.getMinVersion() != null) {- output.writeByte((byte) 1);- Version minVersion = si.getMinVersion();- output.writeInt(minVersion.major);- output.writeInt(minVersion.minor);- output.writeInt(minVersion.bugfix);- } else {- output.writeByte((byte) 0);- }-- assert version.prerelease == 0;- output.writeInt(si.maxDoc());-- output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO));- output.writeMapOfStrings(si.getDiagnostics());- Set<String> files = si.files();- for (String file : files) {- if (!IndexFileNames.parseSegmentName(file).equals(si.name)) {- throw new IllegalArgumentException(- "invalid files: expected segment=" + si.name + ", got=" + files);- }- }- output.writeSetOfStrings(files);- output.writeMapOfStrings(si.getAttributes());-- Sort indexSort = si.getIndexSort();- int numSortFields = indexSort == null ? 0 : indexSort.getSort().length;- output.writeVInt(numSortFields);- for (int i = 0; i < numSortFields; ++i) {- SortField sortField = indexSort.getSort()[i];- SortField.Type sortType = sortField.getType();- output.writeString(sortField.getField());- int sortTypeID;- switch (sortField.getType()) {- case STRING:- sortTypeID = 0;- break;- case LONG:- sortTypeID = 1;- break;- case INT:- sortTypeID = 2;- break;- case DOUBLE:- sortTypeID = 3;- break;- case FLOAT:- sortTypeID = 4;- break;- case CUSTOM:- if (sortField instanceof SortedSetSortField) {- sortTypeID = 5;- sortType = SortField.Type.STRING;- } else if (sortField instanceof SortedNumericSortField) {- sortTypeID = 6;- sortType = ((SortedNumericSortField) sortField).getNumericType();- } else {- throw new IllegalStateException("Unexpected SortedNumericSortField " + sortField);- }- break;- case DOC:- case REWRITEABLE:- case STRING_VAL:- case SCORE:- default:- throw new IllegalStateException("Unexpected sort type: " + sortField.getType());- }- output.writeVInt(sortTypeID);- if (sortTypeID == 5) {- SortedSetSortField ssf = (SortedSetSortField) sortField;- if (ssf.getSelector() == SortedSetSelector.Type.MIN) {- output.writeByte((byte) 0);- } else if (ssf.getSelector() == SortedSetSelector.Type.MAX) {- output.writeByte((byte) 1);- } else if (ssf.getSelector() == SortedSetSelector.Type.MIDDLE_MIN) {- output.writeByte((byte) 2);- } else if (ssf.getSelector() == SortedSetSelector.Type.MIDDLE_MAX) {- output.writeByte((byte) 3);- } else {- throw new IllegalStateException(- "Unexpected SortedSetSelector type: " + ssf.getSelector());- }- } else if (sortTypeID == 6) {- SortedNumericSortField snsf = (SortedNumericSortField) sortField;- if (snsf.getNumericType() == SortField.Type.LONG) {- output.writeByte((byte) 0);- } else if (snsf.getNumericType() == SortField.Type.INT) {- output.writeByte((byte) 1);- } else if (snsf.getNumericType() == SortField.Type.DOUBLE) {- output.writeByte((byte) 2);- } else if (snsf.getNumericType() == SortField.Type.FLOAT) {- output.writeByte((byte) 3);- } else {- throw new IllegalStateException(- "Unexpected SortedNumericSelector type: " + snsf.getNumericType());- }- if (snsf.getSelector() == SortedNumericSelector.Type.MIN) {- output.writeByte((byte) 0);- } else if (snsf.getSelector() == SortedNumericSelector.Type.MAX) {- output.writeByte((byte) 1);- } else {- throw new IllegalStateException(- "Unexpected sorted numeric selector type: " + snsf.getSelector());- }- }- output.writeByte((byte) (sortField.getReverse() ? 0 : 1));-- // write missing value- Object missingValue = sortField.getMissingValue();- if (missingValue == null) {- output.writeByte((byte) 0);- } else {- switch (sortType) {- case STRING:- if (missingValue == SortField.STRING_LAST) {- output.writeByte((byte) 1);- } else if (missingValue == SortField.STRING_FIRST) {- output.writeByte((byte) 2);- } else {- throw new AssertionError(- "unrecognized missing value for STRING field \""- + sortField.getField()- + "\": "- + missingValue);- }- break;- case LONG:- output.writeByte((byte) 1);- output.writeLong(((Long) missingValue).longValue());- break;- case INT:- output.writeByte((byte) 1);- output.writeInt(((Integer) missingValue).intValue());- break;- case DOUBLE:- output.writeByte((byte) 1);- output.writeLong(Double.doubleToLongBits(((Double) missingValue).doubleValue()));- break;- case FLOAT:- output.writeByte((byte) 1);- output.writeInt(Float.floatToIntBits(((Float) missingValue).floatValue()));- break;- case CUSTOM:- case DOC:- case REWRITEABLE:- case STRING_VAL:- case SCORE:- default:- throw new IllegalStateException("Unexpected sort type: " + sortField.getType());- }- }- }- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene70/TestLucene70SegmentInfoFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene70/TestLucene70SegmentInfoFormat.javadeleted file mode 100644index 81482eff07c..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene70/TestLucene70SegmentInfoFormat.java+++ /dev/null@@ -1,43 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--package org.apache.lucene.backward_codecs.lucene70;--import org.apache.lucene.backward_codecs.lucene84.Lucene84RWCodec;-import org.apache.lucene.codecs.Codec;-import org.apache.lucene.tests.index.BaseSegmentInfoFormatTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;-import org.apache.lucene.util.Version;--@Nightly // N-2 formats are only tested on nightly runs-public class TestLucene70SegmentInfoFormat extends BaseSegmentInfoFormatTestCase {-- @Override- protected Version[] getVersions() {- return new Version[] {Version.fromBits(8, 4, 0)};- }-- @Override- protected Codec getCodec() {- return new Lucene84RWCodec();- }-- @Override- protected boolean supportsHasBlocks() {- return false;- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/BaseLucene80DocValuesFormatTestCase.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/BaseLucene80DocValuesFormatTestCase.javadeleted file mode 100644index f77a950f0c1..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/BaseLucene80DocValuesFormatTestCase.java+++ /dev/null@@ -1,872 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene80;--import com.carrotsearch.randomizedtesting.generators.RandomPicks;-import java.io.IOException;-import java.util.ArrayList;-import java.util.Arrays;-import java.util.Collections;-import java.util.HashSet;-import java.util.List;-import java.util.Set;-import java.util.TreeSet;-import java.util.function.LongSupplier;-import java.util.function.Supplier;-import org.apache.lucene.codecs.DocValuesFormat;-import org.apache.lucene.codecs.PostingsFormat;-import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;-import org.apache.lucene.document.BinaryDocValuesField;-import org.apache.lucene.document.Document;-import org.apache.lucene.document.Field;-import org.apache.lucene.document.NumericDocValuesField;-import org.apache.lucene.document.SortedDocValuesField;-import org.apache.lucene.document.SortedNumericDocValuesField;-import org.apache.lucene.document.SortedSetDocValuesField;-import org.apache.lucene.document.StoredField;-import org.apache.lucene.document.StringField;-import org.apache.lucene.index.BinaryDocValues;-import org.apache.lucene.index.DirectoryReader;-import org.apache.lucene.index.DocValues;-import org.apache.lucene.index.IndexReader;-import org.apache.lucene.index.IndexWriter;-import org.apache.lucene.index.IndexWriterConfig;-import org.apache.lucene.index.IndexableField;-import org.apache.lucene.index.LeafReader;-import org.apache.lucene.index.LeafReaderContext;-import org.apache.lucene.index.NumericDocValues;-import org.apache.lucene.index.SerialMergeScheduler;-import org.apache.lucene.index.SortedDocValues;-import org.apache.lucene.index.SortedNumericDocValues;-import org.apache.lucene.index.SortedSetDocValues;-import org.apache.lucene.index.StoredFields;-import org.apache.lucene.index.Term;-import org.apache.lucene.index.Terms;-import org.apache.lucene.index.TermsEnum;-import org.apache.lucene.index.TermsEnum.SeekStatus;-import org.apache.lucene.store.ByteBuffersDataInput;-import org.apache.lucene.store.ByteBuffersDataOutput;-import org.apache.lucene.store.ByteBuffersDirectory;-import org.apache.lucene.store.Directory;-import org.apache.lucene.tests.analysis.MockAnalyzer;-import org.apache.lucene.tests.codecs.asserting.AssertingCodec;-import org.apache.lucene.tests.index.LegacyBaseDocValuesFormatTestCase;-import org.apache.lucene.tests.index.RandomIndexWriter;-import org.apache.lucene.tests.util.TestUtil;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.BytesRefBuilder;-import org.apache.lucene.util.packed.PackedInts;--/** Tests Lucene80DocValuesFormat */-public abstract class BaseLucene80DocValuesFormatTestCase- extends LegacyBaseDocValuesFormatTestCase {-- private static long dirSize(Directory d) throws IOException {- long size = 0;- for (String file : d.listAll()) {- size += d.fileLength(file);- }- return size;- }-- public void testUniqueValuesCompression() throws IOException {- try (final Directory dir = new ByteBuffersDirectory()) {- final IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));- final IndexWriter iwriter = new IndexWriter(dir, iwc);-- final int uniqueValueCount = TestUtil.nextInt(random(), 1, 256);- final List<Long> values = new ArrayList<>();-- final Document doc = new Document();- final NumericDocValuesField dvf = new NumericDocValuesField("dv", 0);- doc.add(dvf);- for (int i = 0; i < 300; ++i) {- final long value;- if (values.size() < uniqueValueCount) {- value = random().nextLong();- values.add(value);- } else {- value = RandomPicks.randomFrom(random(), values);- }- dvf.setLongValue(value);- iwriter.addDocument(doc);- }- iwriter.forceMerge(1);- final long size1 = dirSize(dir);- for (int i = 0; i < 20; ++i) {- dvf.setLongValue(RandomPicks.randomFrom(random(), values));- iwriter.addDocument(doc);- }- iwriter.forceMerge(1);- final long size2 = dirSize(dir);- // make sure the new longs did not cost 8 bytes each- assertTrue(size2 < size1 + 8 * 20);- }- }-- public void testDateCompression() throws IOException {- try (final Directory dir = new ByteBuffersDirectory()) {- final IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));- final IndexWriter iwriter = new IndexWriter(dir, iwc);-- final long base = 13; // prime- final long day = 1000L * 60 * 60 * 24;-- final Document doc = new Document();- final NumericDocValuesField dvf = new NumericDocValuesField("dv", 0);- doc.add(dvf);- for (int i = 0; i < 300; ++i) {- dvf.setLongValue(base + random().nextInt(1000) * day);- iwriter.addDocument(doc);- }- iwriter.forceMerge(1);- final long size1 = dirSize(dir);- for (int i = 0; i < 50; ++i) {- dvf.setLongValue(base + random().nextInt(1000) * day);- iwriter.addDocument(doc);- }- iwriter.forceMerge(1);- final long size2 = dirSize(dir);- // make sure the new longs costed less than if they had only been packed- assertTrue(size2 < size1 + (PackedInts.bitsRequired(day) * 50) / 8);- }- }-- public void testSingleBigValueCompression() throws IOException {- try (final Directory dir = new ByteBuffersDirectory()) {- final IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));- final IndexWriter iwriter = new IndexWriter(dir, iwc);-- final Document doc = new Document();- final NumericDocValuesField dvf = new NumericDocValuesField("dv", 0);- doc.add(dvf);- for (int i = 0; i < 20000; ++i) {- dvf.setLongValue(i & 1023);- iwriter.addDocument(doc);- }- iwriter.forceMerge(1);- final long size1 = dirSize(dir);- dvf.setLongValue(Long.MAX_VALUE);- iwriter.addDocument(doc);- iwriter.forceMerge(1);- final long size2 = dirSize(dir);- // make sure the new value did not grow the bpv for every other value- assertTrue(size2 < size1 + (20000 * (63 - 10)) / 8);- }- }-- // TODO: these big methods can easily blow up some of the other ram-hungry codecs...- // for now just keep them here, as we want to test this for this format.-- public void testSortedSetVariableLengthBigVsStoredFields() throws Exception {- int numIterations = atLeast(1);- for (int i = 0; i < numIterations; i++) {- int numDocs = TEST_NIGHTLY ? atLeast(100) : atLeast(10);- doTestSortedSetVsStoredFields(numDocs, 1, 32766, 16, 100);- }- }-- @Nightly- public void testSortedSetVariableLengthManyVsStoredFields() throws Exception {- int numIterations = atLeast(1);- for (int i = 0; i < numIterations; i++) {- doTestSortedSetVsStoredFields(TestUtil.nextInt(random(), 1024, 2049), 1, 500, 16, 100);- }- }-- public void testSortedVariableLengthBigVsStoredFields() throws Exception {- int numIterations = atLeast(1);- for (int i = 0; i < numIterations; i++) {- doTestSortedVsStoredFields(atLeast(100), 1d, 1, 32766);- }- }-- @Nightly- public void testSortedVariableLengthManyVsStoredFields() throws Exception {- int numIterations = atLeast(1);- for (int i = 0; i < numIterations; i++) {- doTestSortedVsStoredFields(TestUtil.nextInt(random(), 1024, 2049), 1d, 1, 500);- }- }-- @Nightly- public void testTermsEnumFixedWidth() throws Exception {- int numIterations = atLeast(1);- for (int i = 0; i < numIterations; i++) {- doTestTermsEnumRandom(- TestUtil.nextInt(random(), 1025, 5121),- () -> TestUtil.randomSimpleString(random(), 10, 10));- }- }-- @Nightly- public void testTermsEnumVariableWidth() throws Exception {- int numIterations = atLeast(1);- for (int i = 0; i < numIterations; i++) {- doTestTermsEnumRandom(- TestUtil.nextInt(random(), 1025, 5121),- () -> TestUtil.randomSimpleString(random(), 1, 500));- }- }-- @Nightly- public void testTermsEnumRandomMany() throws Exception {- int numIterations = atLeast(1);- for (int i = 0; i < numIterations; i++) {- doTestTermsEnumRandom(- TestUtil.nextInt(random(), 1025, 8121),- () -> TestUtil.randomSimpleString(random(), 1, 500));- }- }-- @Nightly- public void testTermsEnumLongSharedPrefixes() throws Exception {- int numIterations = atLeast(1);- for (int i = 0; i < numIterations; i++) {- doTestTermsEnumRandom(- TestUtil.nextInt(random(), 1025, 5121),- () -> {- char[] chars = new char[random().nextInt(500)];- Arrays.fill(chars, 'a');- if (chars.length > 0) {- chars[random().nextInt(chars.length)] = 'b';- }- return new String(chars);- });- }- }-- public void testSparseDocValuesVsStoredFields() throws Exception {- int numIterations = atLeast(1);- for (int i = 0; i < numIterations; i++) {- doTestSparseDocValuesVsStoredFields();- }- }-- private void doTestSparseDocValuesVsStoredFields() throws Exception {- final long[] values = new long[TestUtil.nextInt(random(), 1, 500)];- for (int i = 0; i < values.length; ++i) {- values[i] = random().nextLong();- }-- Directory dir = newFSDirectory(createTempDir());- IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));- conf.setMergeScheduler(new SerialMergeScheduler());- RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);-- // sparse compression is only enabled if less than 1% of docs have a value- final int avgGap = 100;-- final int numDocs = atLeast(200);- for (int i = random().nextInt(avgGap * 2); i >= 0; --i) {- writer.addDocument(new Document());- }- final int maxNumValuesPerDoc = random().nextBoolean() ? 1 : TestUtil.nextInt(random(), 2, 5);- for (int i = 0; i < numDocs; ++i) {- Document doc = new Document();-- // single-valued- long docValue = values[random().nextInt(values.length)];- doc.add(new NumericDocValuesField("numeric", docValue));- doc.add(new SortedDocValuesField("sorted", new BytesRef(Long.toString(docValue))));- doc.add(new BinaryDocValuesField("binary", new BytesRef(Long.toString(docValue))));- doc.add(new StoredField("value", docValue));-- // multi-valued- final int numValues = TestUtil.nextInt(random(), 1, maxNumValuesPerDoc);- for (int j = 0; j < numValues; ++j) {- docValue = values[random().nextInt(values.length)];- doc.add(new SortedNumericDocValuesField("sorted_numeric", docValue));- doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef(Long.toString(docValue))));- doc.add(new StoredField("values", docValue));- }-- writer.addDocument(doc);-- // add a gap- for (int j = TestUtil.nextInt(random(), 0, avgGap * 2); j >= 0; --j) {- writer.addDocument(new Document());- }- }-- if (random().nextBoolean()) {- writer.forceMerge(1);- }-- final IndexReader indexReader = writer.getReader();- writer.close();-- for (LeafReaderContext context : indexReader.leaves()) {- final LeafReader reader = context.reader();- final NumericDocValues numeric = DocValues.getNumeric(reader, "numeric");-- final SortedDocValues sorted = DocValues.getSorted(reader, "sorted");-- final BinaryDocValues binary = DocValues.getBinary(reader, "binary");-- final SortedNumericDocValues sortedNumeric =- DocValues.getSortedNumeric(reader, "sorted_numeric");-- final SortedSetDocValues sortedSet = DocValues.getSortedSet(reader, "sorted_set");-- StoredFields storedFields = reader.storedFields();- for (int i = 0; i < reader.maxDoc(); ++i) {- final Document doc = storedFields.document(i);- final IndexableField valueField = doc.getField("value");- final Long value = valueField == null ? null : valueField.numericValue().longValue();-- if (value == null) {- assertTrue(numeric.docID() + " vs " + i, numeric.docID() < i);- } else {- assertEquals(i, numeric.nextDoc());- assertEquals(i, binary.nextDoc());- assertEquals(i, sorted.nextDoc());- assertEquals(value.longValue(), numeric.longValue());- assertTrue(sorted.ordValue() >= 0);- assertEquals(new BytesRef(Long.toString(value)), sorted.lookupOrd(sorted.ordValue()));- assertEquals(new BytesRef(Long.toString(value)), binary.binaryValue());- }-- final IndexableField[] valuesFields = doc.getFields("values");- if (valuesFields.length == 0) {- assertTrue(sortedNumeric.docID() + " vs " + i, sortedNumeric.docID() < i);- } else {- final Set<Long> valueSet = new HashSet<>();- for (IndexableField sf : valuesFields) {- valueSet.add(sf.numericValue().longValue());- }-- assertEquals(i, sortedNumeric.nextDoc());- assertEquals(valuesFields.length, sortedNumeric.docValueCount());- for (int j = 0; j < sortedNumeric.docValueCount(); ++j) {- assertTrue(valueSet.contains(sortedNumeric.nextValue()));- }- assertEquals(i, sortedSet.nextDoc());-- assertEquals(valueSet.size(), sortedSet.docValueCount());- for (int j = 0; j < sortedSet.docValueCount(); ++j) {- long ord = sortedSet.nextOrd();- assertTrue(valueSet.contains(Long.parseLong(sortedSet.lookupOrd(ord).utf8ToString())));- }- }- }- }-- indexReader.close();- dir.close();- }-- // TODO: try to refactor this and some termsenum tests into the base class.- // to do this we need to fix the test class to get a DVF not a Codec so we can setup- // the postings format correctly.- private void doTestTermsEnumRandom(int numDocs, Supplier<String> valuesProducer)- throws Exception {- Directory dir = newFSDirectory(createTempDir());- IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));- conf.setMergeScheduler(new SerialMergeScheduler());- // set to duel against a codec which has ordinals:- final PostingsFormat pf = TestUtil.getPostingsFormatWithOrds(random());- final DocValuesFormat dv =- ((PerFieldDocValuesFormat) getCodec().docValuesFormat())- .getDocValuesFormatForField("random_field_name");- conf.setCodec(- new AssertingCodec() {- @Override- public PostingsFormat getPostingsFormatForField(String field) {- return pf;- }-- @Override- public DocValuesFormat getDocValuesFormatForField(String field) {- return dv;- }- });- RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);-- // index some docs- for (int i = 0; i < numDocs; i++) {- Document doc = new Document();- Field idField = new StringField("id", Integer.toString(i), Field.Store.NO);- doc.add(idField);- int numValues = random().nextInt(17);- // create a random list of strings- List<String> values = new ArrayList<>();- for (int v = 0; v < numValues; v++) {- values.add(valuesProducer.get());- }-- // add in any order to the indexed field- ArrayList<String> unordered = new ArrayList<>(values);- Collections.shuffle(unordered, random());- for (String v : values) {- doc.add(newStringField("indexed", v, Field.Store.NO));- }-- // add in any order to the dv field- ArrayList<String> unordered2 = new ArrayList<>(values);- Collections.shuffle(unordered2, random());- for (String v : unordered2) {- doc.add(new SortedSetDocValuesField("dv", new BytesRef(v)));- }-- writer.addDocument(doc);- if (random().nextInt(31) == 0) {- writer.commit();- }- }-- // delete some docs- int numDeletions = random().nextInt(numDocs / 10);- for (int i = 0; i < numDeletions; i++) {- int id = random().nextInt(numDocs);- writer.deleteDocuments(new Term("id", Integer.toString(id)));- }-- // compare per-segment- DirectoryReader ir = writer.getReader();- for (LeafReaderContext context : ir.leaves()) {- LeafReader r = context.reader();- Terms terms = r.terms("indexed");- if (terms != null) {- SortedSetDocValues ssdv = r.getSortedSetDocValues("dv");- assertEquals(terms.size(), ssdv.getValueCount());- TermsEnum expected = terms.iterator();- TermsEnum actual = r.getSortedSetDocValues("dv").termsEnum();- assertEquals(terms.size(), expected, actual);-- doTestSortedSetEnumAdvanceIndependently(ssdv);- }- }- ir.close();-- writer.forceMerge(1);-- // now compare again after the merge- ir = writer.getReader();- LeafReader ar = getOnlyLeafReader(ir);- Terms terms = ar.terms("indexed");- if (terms != null) {- assertEquals(terms.size(), ar.getSortedSetDocValues("dv").getValueCount());- TermsEnum expected = terms.iterator();- TermsEnum actual = ar.getSortedSetDocValues("dv").termsEnum();- assertEquals(terms.size(), expected, actual);- }- ir.close();-- writer.close();- dir.close();- }-- private void assertEquals(long numOrds, TermsEnum expected, TermsEnum actual) throws Exception {- BytesRef ref;-- // sequential next() through all terms- while ((ref = expected.next()) != null) {- assertEquals(ref, actual.next());- assertEquals(expected.ord(), actual.ord());- assertEquals(expected.term(), actual.term());- }- assertNull(actual.next());-- // sequential seekExact(ord) through all terms- for (long i = 0; i < numOrds; i++) {- expected.seekExact(i);- actual.seekExact(i);- assertEquals(expected.ord(), actual.ord());- assertEquals(expected.term(), actual.term());- }-- // sequential seekExact(BytesRef) through all terms- for (long i = 0; i < numOrds; i++) {- expected.seekExact(i);- assertTrue(actual.seekExact(expected.term()));- assertEquals(expected.ord(), actual.ord());- assertEquals(expected.term(), actual.term());- }-- // sequential seekCeil(BytesRef) through all terms- for (long i = 0; i < numOrds; i++) {- expected.seekExact(i);- assertEquals(SeekStatus.FOUND, actual.seekCeil(expected.term()));- assertEquals(expected.ord(), actual.ord());- assertEquals(expected.term(), actual.term());- }-- // random seekExact(ord)- for (long i = 0; i < numOrds; i++) {- long randomOrd = TestUtil.nextLong(random(), 0, numOrds - 1);- expected.seekExact(randomOrd);- actual.seekExact(randomOrd);- assertEquals(expected.ord(), actual.ord());- assertEquals(expected.term(), actual.term());- }-- // random seekExact(BytesRef)- for (long i = 0; i < numOrds; i++) {- long randomOrd = TestUtil.nextLong(random(), 0, numOrds - 1);- expected.seekExact(randomOrd);- actual.seekExact(expected.term());- assertEquals(expected.ord(), actual.ord());- assertEquals(expected.term(), actual.term());- }-- // random seekCeil(BytesRef)- for (long i = 0; i < numOrds; i++) {- BytesRef target = new BytesRef(TestUtil.randomUnicodeString(random()));- SeekStatus expectedStatus = expected.seekCeil(target);- assertEquals(expectedStatus, actual.seekCeil(target));- if (expectedStatus != SeekStatus.END) {- assertEquals(expected.ord(), actual.ord());- assertEquals(expected.term(), actual.term());- }- }- }-- @Nightly- public void testSortedSetAroundBlockSize() throws IOException {- final int frontier = 1 << Lucene80DocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;- for (int maxDoc = frontier - 1; maxDoc <= frontier + 1; ++maxDoc) {- final Directory dir = newDirectory();- IndexWriter w =- new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(newLogMergePolicy()));- ByteBuffersDataOutput out = new ByteBuffersDataOutput();- Document doc = new Document();- SortedSetDocValuesField field1 = new SortedSetDocValuesField("sset", new BytesRef());- doc.add(field1);- SortedSetDocValuesField field2 = new SortedSetDocValuesField("sset", new BytesRef());- doc.add(field2);- for (int i = 0; i < maxDoc; ++i) {- BytesRef s1 = new BytesRef(TestUtil.randomSimpleString(random(), 2));- BytesRef s2 = new BytesRef(TestUtil.randomSimpleString(random(), 2));- field1.setBytesValue(s1);- field2.setBytesValue(s2);- w.addDocument(doc);- Set<BytesRef> set = new TreeSet<>(Arrays.asList(s1, s2));- out.writeVInt(set.size());- for (BytesRef ref : set) {- out.writeVInt(ref.length);- out.writeBytes(ref.bytes, ref.offset, ref.length);- }- }-- w.forceMerge(1);- DirectoryReader r = DirectoryReader.open(w);- w.close();- LeafReader sr = getOnlyLeafReader(r);- assertEquals(maxDoc, sr.maxDoc());- SortedSetDocValues values = sr.getSortedSetDocValues("sset");- assertNotNull(values);- ByteBuffersDataInput in = out.toDataInput();- BytesRefBuilder b = new BytesRefBuilder();- for (int i = 0; i < maxDoc; ++i) {- assertEquals(i, values.nextDoc());- final int numValues = in.readVInt();- assertEquals(numValues, values.docValueCount());-- for (int j = 0; j < numValues; ++j) {- b.setLength(in.readVInt());- b.grow(b.length());- in.readBytes(b.bytes(), 0, b.length());- assertEquals(b.get(), values.lookupOrd(values.nextOrd()));- }- }- r.close();- dir.close();- }- }-- @Nightly- public void testSortedNumericAroundBlockSize() throws IOException {- final int frontier = 1 << Lucene80DocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;- for (int maxDoc = frontier - 1; maxDoc <= frontier + 1; ++maxDoc) {- final Directory dir = newDirectory();- IndexWriter w =- new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(newLogMergePolicy()));- ByteBuffersDataOutput buffer = new ByteBuffersDataOutput();-- Document doc = new Document();- SortedNumericDocValuesField field1 = new SortedNumericDocValuesField("snum", 0L);- doc.add(field1);- SortedNumericDocValuesField field2 = new SortedNumericDocValuesField("snum", 0L);- doc.add(field2);- for (int i = 0; i < maxDoc; ++i) {- long s1 = random().nextInt(100);- long s2 = random().nextInt(100);- field1.setLongValue(s1);- field2.setLongValue(s2);- w.addDocument(doc);- buffer.writeVLong(Math.min(s1, s2));- buffer.writeVLong(Math.max(s1, s2));- }-- w.forceMerge(1);- DirectoryReader r = DirectoryReader.open(w);- w.close();- LeafReader sr = getOnlyLeafReader(r);- assertEquals(maxDoc, sr.maxDoc());- SortedNumericDocValues values = sr.getSortedNumericDocValues("snum");- assertNotNull(values);- ByteBuffersDataInput dataInput = buffer.toDataInput();- for (int i = 0; i < maxDoc; ++i) {- assertEquals(i, values.nextDoc());- assertEquals(2, values.docValueCount());- assertEquals(dataInput.readVLong(), values.nextValue());- assertEquals(dataInput.readVLong(), values.nextValue());- }- r.close();- dir.close();- }- }-- @Nightly- public void testSortedNumericBlocksOfVariousBitsPerValue() throws Exception {- doTestSortedNumericBlocksOfVariousBitsPerValue(() -> TestUtil.nextInt(random(), 1, 3));- }-- @Nightly- public void testSparseSortedNumericBlocksOfVariousBitsPerValue() throws Exception {- doTestSortedNumericBlocksOfVariousBitsPerValue(() -> TestUtil.nextInt(random(), 0, 2));- }-- @Nightly- public void testNumericBlocksOfVariousBitsPerValue() throws Exception {- doTestSparseNumericBlocksOfVariousBitsPerValue(1);- }-- @Nightly- public void testSparseNumericBlocksOfVariousBitsPerValue() throws Exception {- doTestSparseNumericBlocksOfVariousBitsPerValue(random().nextDouble());- }-- // The LUCENE-8585 jump-tables enables O(1) skipping of IndexedDISI blocks, DENSE block lookup- // and numeric multi blocks. This test focuses on testing these jumps.- @Nightly- public void testNumericFieldJumpTables() throws Exception {- // IndexedDISI block skipping only activated if target >= current+2, so we need at least 5- // blocks to- // trigger consecutive block skips- final int maxDoc = atLeast(5 * 65536);-- Directory dir = newDirectory();- IndexWriter iw = createFastIndexWriter(dir, maxDoc);-- Field idField = newStringField("id", "", Field.Store.NO);- Field storedField = newStringField("stored", "", Field.Store.YES);- Field dvField = new NumericDocValuesField("dv", 0);-- for (int i = 0; i < maxDoc; i++) {- Document doc = new Document();- idField.setStringValue(Integer.toBinaryString(i));- doc.add(idField);- if (random().nextInt(100) > 10) { // Skip 10% to make DENSE blocks- int value = random().nextInt(100000);- storedField.setStringValue(Integer.toString(value));- doc.add(storedField);- dvField.setLongValue(value);- doc.add(dvField);- }- iw.addDocument(doc);- }- iw.flush();- iw.forceMerge(1, true); // Single segment to force large enough structures- iw.commit();- iw.close();-- assertDVIterate(dir);- assertDVAdvance(- dir, rarely() ? 1 : 7); // 1 is heavy (~20 s), so we do it rarely. 7 is a lot faster (8 s)-- dir.close();- }-- private IndexWriter createFastIndexWriter(Directory dir, int maxBufferedDocs) throws IOException {- IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));- conf.setMaxBufferedDocs(maxBufferedDocs);- conf.setRAMBufferSizeMB(-1);- conf.setMergePolicy(newLogMergePolicy());- conf.getCodec().compoundFormat().setShouldUseCompoundFile(random().nextBoolean());- return new IndexWriter(dir, conf);- }-- private static LongSupplier blocksOfVariousBPV() {- final long mul = TestUtil.nextInt(random(), 1, 100);- final long min = random().nextInt();- return new LongSupplier() {- int i = Lucene80DocValuesFormat.NUMERIC_BLOCK_SIZE;- int maxDelta;-- @Override- public long getAsLong() {- if (i == Lucene80DocValuesFormat.NUMERIC_BLOCK_SIZE) {- maxDelta = 1 << random().nextInt(5);- i = 0;- }- i++;- return min + mul * random().nextInt(maxDelta);- }- };- }-- private void doTestSortedNumericBlocksOfVariousBitsPerValue(LongSupplier counts)- throws Exception {- Directory dir = newDirectory();- IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));- conf.setMaxBufferedDocs(atLeast(Lucene80DocValuesFormat.NUMERIC_BLOCK_SIZE));- conf.setRAMBufferSizeMB(-1);- // so Lucene docids are predictable / stay in order- conf.setMergePolicy(newLogMergePolicy());- conf.getCodec().compoundFormat().setShouldUseCompoundFile(random().nextBoolean());- IndexWriter writer = new IndexWriter(dir, conf);-- final int numDocs = atLeast(Lucene80DocValuesFormat.NUMERIC_BLOCK_SIZE * 3);- final LongSupplier values = blocksOfVariousBPV();- List<long[]> writeDocValues = new ArrayList<>();- for (int i = 0; i < numDocs; i++) {- Document doc = new Document();-- int valueCount = (int) counts.getAsLong();- long[] valueArray = new long[valueCount];- for (int j = 0; j < valueCount; j++) {- long value = values.getAsLong();- valueArray[j] = value;- doc.add(new SortedNumericDocValuesField("dv", value));- }- Arrays.sort(valueArray);- writeDocValues.add(valueArray);- for (int j = 0; j < valueCount; j++) {- doc.add(new StoredField("stored", Long.toString(valueArray[j])));- }- writer.addDocument(doc);- if (random().nextInt(31) == 0) {- writer.commit();- }- }- writer.forceMerge(1);-- writer.close();-- // compare- DirectoryReader ir = DirectoryReader.open(dir);- TestUtil.checkReader(ir);- for (LeafReaderContext context : ir.leaves()) {- LeafReader r = context.reader();- SortedNumericDocValues docValues = DocValues.getSortedNumeric(r, "dv");- StoredFields storedFields = r.storedFields();- for (int i = 0; i < r.maxDoc(); i++) {- if (i > docValues.docID()) {- docValues.nextDoc();- }- String[] expectedStored = storedFields.document(i).getValues("stored");- if (i < docValues.docID()) {- assertEquals(0, expectedStored.length);- } else {- long[] readValueArray = new long[docValues.docValueCount()];- String[] actualDocValue = new String[docValues.docValueCount()];- for (int j = 0; j < docValues.docValueCount(); ++j) {- long actualDV = docValues.nextValue();- readValueArray[j] = actualDV;- actualDocValue[j] = Long.toString(readValueArray[j]);- }- long[] writeValueArray = writeDocValues.get(i);- // compare write values and read values- assertArrayEquals(readValueArray, writeValueArray);-- // compare dv and stored values- assertArrayEquals(expectedStored, actualDocValue);- }- }- }- ir.close();- dir.close();- }-- private void doTestSparseNumericBlocksOfVariousBitsPerValue(double density) throws Exception {- Directory dir = newDirectory();- IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));- conf.setMaxBufferedDocs(atLeast(Lucene80DocValuesFormat.NUMERIC_BLOCK_SIZE));- conf.setRAMBufferSizeMB(-1);- conf.setMergePolicy(newLogMergePolicy());- conf.getCodec().compoundFormat().setShouldUseCompoundFile(random().nextBoolean());- IndexWriter writer = new IndexWriter(dir, conf);- Document doc = new Document();- Field storedField = newStringField("stored", "", Field.Store.YES);- Field dvField = new NumericDocValuesField("dv", 0);- doc.add(storedField);- doc.add(dvField);-- final int numDocs = atLeast(Lucene80DocValuesFormat.NUMERIC_BLOCK_SIZE * 3);- final LongSupplier longs = blocksOfVariousBPV();- for (int i = 0; i < numDocs; i++) {- if (random().nextDouble() > density) {- writer.addDocument(new Document());- continue;- }- long value = longs.getAsLong();- storedField.setStringValue(Long.toString(value));- dvField.setLongValue(value);- writer.addDocument(doc);- }-- writer.forceMerge(1);-- writer.close();-- // compare- assertDVIterate(dir);- assertDVAdvance(- dir, 1); // Tests all jump-lengths from 1 to maxDoc (quite slow ~= 1 minute for 200K docs)-- dir.close();- }-- // Tests that advanceExact does not change the outcome- private void assertDVAdvance(Directory dir, int jumpStep) throws IOException {- DirectoryReader ir = DirectoryReader.open(dir);- TestUtil.checkReader(ir);- for (LeafReaderContext context : ir.leaves()) {- LeafReader r = context.reader();- StoredFields storedFields = r.storedFields();-- for (int jump = jumpStep; jump < r.maxDoc(); jump += jumpStep) {- // Create a new instance each time to ensure jumps from the beginning- NumericDocValues docValues = DocValues.getNumeric(r, "dv");- for (int docID = 0; docID < r.maxDoc(); docID += jump) {- String base =- "document #"- + docID- + "/"- + r.maxDoc()- + ", jumping "- + jump- + " from #"- + (docID - jump);- String storedValue = storedFields.document(docID).get("stored");- if (storedValue == null) {- assertFalse("There should be no DocValue for " + base, docValues.advanceExact(docID));- } else {- assertTrue("There should be a DocValue for " + base, docValues.advanceExact(docID));- assertEquals(- "The doc value should be correct for " + base,- Long.parseLong(storedValue),- docValues.longValue());- }- }- }- }- ir.close();- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/Lucene80NormsConsumer.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/Lucene80NormsConsumer.javadeleted file mode 100644index aa16a8916ee..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/Lucene80NormsConsumer.java+++ /dev/null@@ -1,177 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene80;--import static org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat.VERSION_CURRENT;--import java.io.IOException;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.NormsConsumer;-import org.apache.lucene.codecs.NormsProducer;-import org.apache.lucene.index.FieldInfo;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.NumericDocValues;-import org.apache.lucene.index.SegmentWriteState;-import org.apache.lucene.search.DocIdSetIterator;-import org.apache.lucene.store.IndexOutput;-import org.apache.lucene.util.IOUtils;--/** Writer for {@link Lucene80NormsFormat} */-final class Lucene80NormsConsumer extends NormsConsumer {- IndexOutput data, meta;- final int maxDoc;-- Lucene80NormsConsumer(- SegmentWriteState state,- String dataCodec,- String dataExtension,- String metaCodec,- String metaExtension)- throws IOException {- boolean success = false;- try {- String dataName =- IndexFileNames.segmentFileName(- state.segmentInfo.name, state.segmentSuffix, dataExtension);- data = EndiannessReverserUtil.createOutput(state.directory, dataName, state.context);- CodecUtil.writeIndexHeader(- data, dataCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);- String metaName =- IndexFileNames.segmentFileName(- state.segmentInfo.name, state.segmentSuffix, metaExtension);- meta = EndiannessReverserUtil.createOutput(state.directory, metaName, state.context);- CodecUtil.writeIndexHeader(- meta, metaCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);- maxDoc = state.segmentInfo.maxDoc();- success = true;- } finally {- if (!success) {- IOUtils.closeWhileHandlingException(this);- }- }- }-- @Override- public void close() throws IOException {- boolean success = false;- try {- if (meta != null) {- meta.writeInt(-1); // write EOF marker- CodecUtil.writeFooter(meta); // write checksum- }- if (data != null) {- CodecUtil.writeFooter(data); // write checksum- }- success = true;- } finally {- if (success) {- IOUtils.close(data, meta);- } else {- IOUtils.closeWhileHandlingException(data, meta);- }- meta = data = null;- }- }-- @Override- public void addNormsField(FieldInfo field, NormsProducer normsProducer) throws IOException {- NumericDocValues values = normsProducer.getNorms(field);- int numDocsWithValue = 0;- long min = Long.MAX_VALUE;- long max = Long.MIN_VALUE;- for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {- numDocsWithValue++;- long v = values.longValue();- min = Math.min(min, v);- max = Math.max(max, v);- }- assert numDocsWithValue <= maxDoc;-- meta.writeInt(field.number);-- if (numDocsWithValue == 0) {- meta.writeLong(-2); // docsWithFieldOffset- meta.writeLong(0L); // docsWithFieldLength- meta.writeShort((short) -1); // jumpTableEntryCount- meta.writeByte((byte) -1); // denseRankPower- } else if (numDocsWithValue == maxDoc) {- meta.writeLong(-1); // docsWithFieldOffset- meta.writeLong(0L); // docsWithFieldLength- meta.writeShort((short) -1); // jumpTableEntryCount- meta.writeByte((byte) -1); // denseRankPower- } else {- long offset = data.getFilePointer();- meta.writeLong(offset); // docsWithFieldOffset- values = normsProducer.getNorms(field);- final short jumpTableEntryCount =- IndexedDISI.writeBitSet(values, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER);- meta.writeLong(data.getFilePointer() - offset); // docsWithFieldLength- meta.writeShort(jumpTableEntryCount);- meta.writeByte(IndexedDISI.DEFAULT_DENSE_RANK_POWER);- }-- meta.writeInt(numDocsWithValue);- int numBytesPerValue = numBytesPerValue(min, max);-- meta.writeByte((byte) numBytesPerValue);- if (numBytesPerValue == 0) {- meta.writeLong(min);- } else {- meta.writeLong(data.getFilePointer()); // normsOffset- values = normsProducer.getNorms(field);- writeValues(values, numBytesPerValue, data);- }- }-- private int numBytesPerValue(long min, long max) {- if (min >= max) {- return 0;- } else if (min >= Byte.MIN_VALUE && max <= Byte.MAX_VALUE) {- return 1;- } else if (min >= Short.MIN_VALUE && max <= Short.MAX_VALUE) {- return 2;- } else if (min >= Integer.MIN_VALUE && max <= Integer.MAX_VALUE) {- return 4;- } else {- return 8;- }- }-- private void writeValues(NumericDocValues values, int numBytesPerValue, IndexOutput out)- throws IOException, AssertionError {- for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {- long value = values.longValue();- switch (numBytesPerValue) {- case 1:- out.writeByte((byte) value);- break;- case 2:- out.writeShort((short) value);- break;- case 4:- out.writeInt((int) value);- break;- case 8:- out.writeLong(value);- break;- default:- throw new AssertionError();- }- }- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/Lucene80RWNormsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/Lucene80RWNormsFormat.javadeleted file mode 100644index 1eee0f256f9..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/Lucene80RWNormsFormat.java+++ /dev/null@@ -1,30 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene80;--import java.io.IOException;-import org.apache.lucene.codecs.NormsConsumer;-import org.apache.lucene.index.SegmentWriteState;--public class Lucene80RWNormsFormat extends Lucene80NormsFormat {-- @Override- public NormsConsumer normsConsumer(SegmentWriteState state) throws IOException {- return new Lucene80NormsConsumer(- state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestBestCompressionLucene80DocValuesFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestBestCompressionLucene80DocValuesFormat.javadeleted file mode 100644index 93ca2fe3cee..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestBestCompressionLucene80DocValuesFormat.java+++ /dev/null@@ -1,35 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene80;--import org.apache.lucene.codecs.Codec;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;-import org.apache.lucene.tests.util.TestUtil;--/** Tests Lucene80DocValuesFormat */-@Nightly // N-2 formats are only tested on nightly runs-public class TestBestCompressionLucene80DocValuesFormat- extends BaseLucene80DocValuesFormatTestCase {- private final Codec codec =- TestUtil.alwaysDocValuesFormat(- new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION));-- @Override- protected Codec getCodec() {- return codec;- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestBestSpeedLucene80DocValuesFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestBestSpeedLucene80DocValuesFormat.javadeleted file mode 100644index 2bdb660ca70..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestBestSpeedLucene80DocValuesFormat.java+++ /dev/null@@ -1,34 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene80;--import org.apache.lucene.codecs.Codec;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;-import org.apache.lucene.tests.util.TestUtil;--/** Tests Lucene80DocValuesFormat */-@Nightly // N-2 formats are only tested on nightly runs-public class TestBestSpeedLucene80DocValuesFormat extends BaseLucene80DocValuesFormatTestCase {- private final Codec codec =- TestUtil.alwaysDocValuesFormat(- new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_SPEED));-- @Override- protected Codec getCodec() {- return codec;- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestDocValuesCompression.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestDocValuesCompression.javadeleted file mode 100644index b1ab62c3595..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestDocValuesCompression.java+++ /dev/null@@ -1,320 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene80;--import java.io.IOException;-import java.util.ArrayList;-import java.util.Collections;-import java.util.HashSet;-import java.util.List;-import java.util.Set;-import org.apache.lucene.analysis.Analyzer;-import org.apache.lucene.analysis.standard.StandardAnalyzer;-import org.apache.lucene.codecs.Codec;-import org.apache.lucene.document.Document;-import org.apache.lucene.document.Field;-import org.apache.lucene.document.SortedDocValuesField;-import org.apache.lucene.document.SortedSetDocValuesField;-import org.apache.lucene.document.StringField;-import org.apache.lucene.index.DirectoryReader;-import org.apache.lucene.index.IndexWriter;-import org.apache.lucene.index.IndexWriterConfig;-import org.apache.lucene.index.LeafReader;-import org.apache.lucene.index.SortedDocValues;-import org.apache.lucene.index.SortedSetDocValues;-import org.apache.lucene.store.Directory;-import org.apache.lucene.tests.util.LuceneTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;-import org.apache.lucene.tests.util.TestUtil;-import org.apache.lucene.util.BytesRef;--@Nightly // N-2 formats are only tested on nightly runs-public class TestDocValuesCompression extends LuceneTestCase {- private final Codec bestSpeed =- TestUtil.alwaysDocValuesFormat(- new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_SPEED));-- private final Codec bestCompression =- TestUtil.alwaysDocValuesFormat(- new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION));-- public void testTermsDictCompressionForLowCardinalityFields() throws IOException {- final int CARDINALITY = Lucene80DocValuesFormat.TERMS_DICT_BLOCK_COMPRESSION_THRESHOLD - 1;- Set<String> valuesSet = new HashSet<>();- for (int i = 0; i < CARDINALITY; ++i) {- final int length = TestUtil.nextInt(random(), 10, 30);- String value = TestUtil.randomSimpleString(random(), length);- valuesSet.add(value);- }-- List<String> values = new ArrayList<>(valuesSet);- long sizeForBestSpeed = writeAndGetDocValueFileSize(bestSpeed, values);- long sizeForBestCompression = writeAndGetDocValueFileSize(bestCompression, values);-- // Ensure terms dict data was not compressed for low-cardinality fields.- assertEquals(sizeForBestSpeed, sizeForBestCompression);- }-- public void testTermsDictCompressionForHighCardinalityFields() throws IOException {- final int CARDINALITY = Lucene80DocValuesFormat.TERMS_DICT_BLOCK_COMPRESSION_THRESHOLD << 1;- Set<String> valuesSet = new HashSet<>();- for (int i = 0; i < CARDINALITY; ++i) {- final int length = TestUtil.nextInt(random(), 10, 30);- String value = TestUtil.randomSimpleString(random(), length);- // Add common suffix for better compression ratio.- valuesSet.add(value + "_CommonPartBetterForCompression");- }-- List<String> values = new ArrayList<>(valuesSet);- long sizeForBestSpeed = writeAndGetDocValueFileSize(bestSpeed, values);- long sizeForBestCompression = writeAndGetDocValueFileSize(bestCompression, values);-- // Compression happened.- assertTrue(sizeForBestCompression < sizeForBestSpeed);- }-- public void testReseekAfterSkipDecompression() throws IOException {- final int CARDINALITY = (Lucene80DocValuesFormat.TERMS_DICT_BLOCK_LZ4_SIZE << 1) + 11;- Set<String> valueSet = HashSet.newHashSet(CARDINALITY);- for (int i = 0; i < CARDINALITY; i++) {- valueSet.add(TestUtil.randomSimpleString(random(), 64));- }- List<String> values = new ArrayList<>(valueSet);- Collections.sort(values);- // Create one non-existent value just between block-1 and block-2.- String nonexistentValue =- values.get(Lucene80DocValuesFormat.TERMS_DICT_BLOCK_LZ4_SIZE - 1)- + TestUtil.randomSimpleString(random(), 64, 128);- int docValues = values.size();-- try (Directory directory = newDirectory()) {- Analyzer analyzer = new StandardAnalyzer();- IndexWriterConfig config = new IndexWriterConfig(analyzer);- config.setCodec(bestCompression);- config.setUseCompoundFile(false);- IndexWriter writer = new IndexWriter(directory, config);- for (int i = 0; i < 280; i++) {- Document doc = new Document();- doc.add(new StringField("id", "Doc" + i, Field.Store.NO));- doc.add(new SortedDocValuesField("sdv", new BytesRef(values.get(i % docValues))));- writer.addDocument(doc);- }- writer.commit();- writer.forceMerge(1);- DirectoryReader dReader = DirectoryReader.open(writer);- writer.close();-- LeafReader reader = getOnlyLeafReader(dReader);- // Check values count.- SortedDocValues ssdvMulti = reader.getSortedDocValues("sdv");- assertEquals(docValues, ssdvMulti.getValueCount());-- // Seek to first block.- int ord1 = ssdvMulti.lookupTerm(new BytesRef(values.get(0)));- assertTrue(ord1 >= 0);- int ord2 = ssdvMulti.lookupTerm(new BytesRef(values.get(1)));- assertTrue(ord2 >= ord1);- // Ensure re-seek logic is correct after skip-decompression.- int nonexistentOrd2 = ssdvMulti.lookupTerm(new BytesRef(nonexistentValue));- assertTrue(nonexistentOrd2 < 0);- dReader.close();- }- }-- public void testLargeTermsCompression() throws IOException {- final int CARDINALITY = Lucene80DocValuesFormat.TERMS_DICT_BLOCK_COMPRESSION_THRESHOLD << 1;- Set<String> valuesSet = new HashSet<>();- for (int i = 0; i < CARDINALITY; ++i) {- final int length = TestUtil.nextInt(random(), 512, 1024);- valuesSet.add(TestUtil.randomSimpleString(random(), length));- }- int valuesCount = valuesSet.size();- List<String> values = new ArrayList<>(valuesSet);-- try (Directory directory = newDirectory()) {- Analyzer analyzer = new StandardAnalyzer();- IndexWriterConfig config = new IndexWriterConfig(analyzer);- config.setCodec(bestCompression);- config.setUseCompoundFile(false);- IndexWriter writer = new IndexWriter(directory, config);- for (int i = 0; i < 256; i++) {- Document doc = new Document();- doc.add(new StringField("id", "Doc" + i, Field.Store.NO));- doc.add(new SortedDocValuesField("sdv", new BytesRef(values.get(i % valuesCount))));- writer.addDocument(doc);- }- writer.commit();- writer.forceMerge(1);- DirectoryReader ireader = DirectoryReader.open(writer);- writer.close();-- LeafReader reader = getOnlyLeafReader(ireader);- // Check values count.- SortedDocValues ssdvMulti = reader.getSortedDocValues("sdv");- assertEquals(valuesCount, ssdvMulti.getValueCount());- ireader.close();- }- }-- // Ensure the old segment can be merged together with the new compressed segment.- public void testMergeWithUncompressedSegment() throws IOException {- final int CARDINALITY = Lucene80DocValuesFormat.TERMS_DICT_BLOCK_COMPRESSION_THRESHOLD << 1;- Set<String> valuesSet = new HashSet<>();- for (int i = 0; i < CARDINALITY; ++i) {- final int length = TestUtil.nextInt(random(), 10, 30);- // Add common suffix for better compression ratio.- valuesSet.add(TestUtil.randomSimpleString(random(), length));- }- List<String> values = new ArrayList<>(valuesSet);- int valuesCount = values.size();-- try (Directory directory = newDirectory()) {- // 1. Write 256 documents without terms dict compression.- Analyzer analyzer = new StandardAnalyzer();- IndexWriterConfig config = new IndexWriterConfig(analyzer);- config.setCodec(bestSpeed);- config.setUseCompoundFile(false);- IndexWriter writer = new IndexWriter(directory, config);- for (int i = 0; i < 256; i++) {- Document doc = new Document();- doc.add(new StringField("id", "Doc" + i, Field.Store.NO));- doc.add(new SortedSetDocValuesField("ssdv", new BytesRef(values.get(i % valuesCount))));- doc.add(- new SortedSetDocValuesField("ssdv", new BytesRef(values.get((i + 1) % valuesCount))));- doc.add(- new SortedSetDocValuesField("ssdv", new BytesRef(values.get((i + 2) % valuesCount))));- doc.add(new SortedDocValuesField("sdv", new BytesRef(values.get(i % valuesCount))));- writer.addDocument(doc);- }- writer.commit();- DirectoryReader ireader = DirectoryReader.open(writer);- assertEquals(256, ireader.numDocs());- LeafReader reader = getOnlyLeafReader(ireader);- SortedSetDocValues ssdv = reader.getSortedSetDocValues("ssdv");- assertEquals(valuesCount, ssdv.getValueCount());- SortedDocValues sdv = reader.getSortedDocValues("sdv");- assertEquals(valuesCount, sdv.getValueCount());- ireader.close();- writer.close();-- // 2. Add another 100 documents, and enabling terms dict compression.- config = new IndexWriterConfig(analyzer);- config.setCodec(bestCompression);- config.setUseCompoundFile(false);- writer = new IndexWriter(directory, config);- // Add 2 new values.- valuesSet.add(TestUtil.randomSimpleString(random(), 10));- valuesSet.add(TestUtil.randomSimpleString(random(), 10));- values = new ArrayList<>(valuesSet);- valuesCount = valuesSet.size();-- for (int i = 256; i < 356; i++) {- Document doc = new Document();- doc.add(new StringField("id", "Doc" + i, Field.Store.NO));- doc.add(new SortedSetDocValuesField("ssdv", new BytesRef(values.get(i % valuesCount))));- doc.add(new SortedDocValuesField("sdv", new BytesRef(values.get(i % valuesCount))));- writer.addDocument(doc);- }- writer.commit();- writer.forceMerge(1);- ireader = DirectoryReader.open(writer);- assertEquals(356, ireader.numDocs());- reader = getOnlyLeafReader(ireader);- ssdv = reader.getSortedSetDocValues("ssdv");- assertEquals(valuesCount, ssdv.getValueCount());- ireader.close();- writer.close();- }- }-- private static long writeAndGetDocValueFileSize(Codec codec, List<String> values)- throws IOException {- int valuesCount = values.size();- long dvdFileSize = -1;- try (Directory directory = newDirectory()) {- Analyzer analyzer = new StandardAnalyzer();- IndexWriterConfig config = new IndexWriterConfig(analyzer);- config.setCodec(codec);- config.setUseCompoundFile(false);- IndexWriter writer = new IndexWriter(directory, config);- for (int i = 0; i < 256; i++) {- Document doc = new Document();- doc.add(new StringField("id", "Doc" + i, Field.Store.NO));- // Multi value sorted-set field.- doc.add(- new SortedSetDocValuesField("ssdv_multi_", new BytesRef(values.get(i % valuesCount))));- doc.add(- new SortedSetDocValuesField(- "ssdv_multi_", new BytesRef(values.get((i + 1) % valuesCount))));- doc.add(- new SortedSetDocValuesField(- "ssdv_multi_", new BytesRef(values.get((i + 2) % valuesCount))));- // Single value sorted-set field.- doc.add(- new SortedSetDocValuesField("ssdv_single_", new BytesRef(values.get(i % valuesCount))));- // Sorted field.- doc.add(new SortedDocValuesField("sdv", new BytesRef(values.get(i % valuesCount))));- writer.addDocument(doc);- }- writer.commit();- writer.forceMerge(1);- DirectoryReader ireader = DirectoryReader.open(writer);- writer.close();-- LeafReader reader = getOnlyLeafReader(ireader);- // Check values count.- SortedSetDocValues ssdvMulti = reader.getSortedSetDocValues("ssdv_multi_");- assertEquals(valuesCount, ssdvMulti.getValueCount());- for (int i = 0; i < valuesCount; i++) {- BytesRef term = ssdvMulti.lookupOrd(i);- assertTrue(term.bytes.length > 0);- }- for (int i = 0; i < valuesCount; i++) {- for (int j = 0; j < 3; j++) {- assertTrue(ssdvMulti.lookupTerm(new BytesRef(values.get((i + j) % valuesCount))) >= 0);- }- }-- SortedSetDocValues ssdvSingle = reader.getSortedSetDocValues("ssdv_single_");- assertEquals(valuesCount, ssdvSingle.getValueCount());- for (int i = 0; i < valuesCount; i++) {- assertTrue(ssdvSingle.lookupTerm(new BytesRef(values.get(i % valuesCount))) >= 0);- }-- SortedDocValues sdv = reader.getSortedDocValues("sdv");- assertEquals(valuesCount, sdv.getValueCount());- for (int i = 0; i < valuesCount; i++) {- assertTrue(sdv.lookupTerm(new BytesRef(values.get(i % valuesCount))) >= 0);- }-- dvdFileSize = docValueFileSize(directory);- assertTrue(dvdFileSize > 0);- ireader.close();- }-- return dvdFileSize;- }-- static long docValueFileSize(Directory d) throws IOException {- for (String file : d.listAll()) {- if (file.endsWith(Lucene80DocValuesFormat.DATA_EXTENSION)) {- return d.fileLength(file);- }- }- return -1;- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestIndexedDISI.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestIndexedDISI.javadeleted file mode 100644index 800c31d5e5d..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestIndexedDISI.java+++ /dev/null@@ -1,588 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene80;--import java.io.IOException;-import java.util.Random;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.search.DocIdSetIterator;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.store.IndexOutput;-import org.apache.lucene.store.RandomAccessInput;-import org.apache.lucene.tests.util.LuceneTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;-import org.apache.lucene.tests.util.TestUtil;-import org.apache.lucene.util.BitSet;-import org.apache.lucene.util.BitSetIterator;-import org.apache.lucene.util.FixedBitSet;-import org.apache.lucene.util.SparseFixedBitSet;--@Nightly // N-2 formats are only tested on nightly runs-public class TestIndexedDISI extends LuceneTestCase {-- public void testEmpty() throws IOException {- int maxDoc = TestUtil.nextInt(random(), 1, 100000);- BitSet set = new SparseFixedBitSet(maxDoc);- try (Directory dir = newDirectory()) {- doTest(set, dir);- }- }-- // EMPTY blocks are special with regard to jumps as they have size 0- @Nightly- public void testEmptyBlocks() throws IOException {- final int B = 65536;- int maxDoc = B * 11;- BitSet set = new SparseFixedBitSet(maxDoc);- // block 0: EMPTY- set.set(B + 5); // block 1: SPARSE- // block 2: EMPTY- // block 3: EMPTY- set.set(B * 4 + 5); // block 4: SPARSE-- for (int i = 0; i < B; i++) {- set.set(B * 6 + i); // block 6: ALL- }- for (int i = 0; i < B; i += 3) {- set.set(B * 7 + i); // block 7: DENSE- }- for (int i = 0; i < B; i++) {- if (i != 32768) {- set.set(B * 8 + i); // block 8: DENSE (all-1)- }- }- // block 9-11: EMPTY-- try (Directory dir = newDirectory()) {- doTestAllSingleJump(set, dir);- }-- // Change the first block to DENSE to see if jump-tables sets to position 0- set.set(0);- try (Directory dir = newDirectory()) {- doTestAllSingleJump(set, dir);- }- }-- // EMPTY blocks are special with regard to jumps as they have size 0- public void testLastEmptyBlocks() throws IOException {- final int B = 65536;- int maxDoc = B * 3;- BitSet set = new SparseFixedBitSet(maxDoc);- for (int docID = 0; docID < B * 2; docID++) { // first 2 blocks are ALL- set.set(docID);- }- // Last block is EMPTY-- try (Directory dir = newDirectory()) {- doTestAllSingleJump(set, dir);- assertAdvanceBeyondEnd(set, dir);- }- }-- // Checks that advance after the end of the blocks has been reached has the correct behaviour- private void assertAdvanceBeyondEnd(BitSet set, Directory dir) throws IOException {- final int cardinality = set.cardinality();- final byte denseRankPower = 9; // Not tested here so fixed to isolate factors- int jumpTableentryCount;- try (IndexOutput out = EndiannessReverserUtil.createOutput(dir, "bar", IOContext.DEFAULT)) {- jumpTableentryCount =- IndexedDISI.writeBitSet(new BitSetIterator(set, cardinality), out, denseRankPower);- }-- try (IndexInput in = EndiannessReverserUtil.openInput(dir, "bar", IOContext.DEFAULT)) {- BitSetIterator disi2 = new BitSetIterator(set, cardinality);- int doc = disi2.docID();- int index = 0;- while (doc < cardinality) {- doc = disi2.nextDoc();- index++;- }-- IndexedDISI disi =- new IndexedDISI(in, 0L, in.length(), jumpTableentryCount, denseRankPower, cardinality);- // Advance 1 docID beyond end- assertFalse(- "There should be no set bit beyond the valid docID range",- disi.advanceExact(set.length()));- disi.advance(- doc); // Should be the special docID signifyin NO_MORE_DOCS from the BitSetIterator- assertEquals(- "The index when advancing beyond the last defined docID should be correct",- index,- disi.index() + 1); // disi.index()+1 as the while-loop also counts the NO_MORE_DOCS- }- }-- // TODO: can this be toned down?- @Nightly- public void testRandomBlocks() throws IOException {- final int BLOCKS = 5;- BitSet set = createSetWithRandomBlocks(BLOCKS);- try (Directory dir = newDirectory()) {- doTestAllSingleJump(set, dir);- }- }-- // When doing merges in Lucene80NormsProducer, IndexedDISI are created from slices where the- // offset is not 0- public void testPositionNotZero() throws IOException {- final int BLOCKS = 10;- final byte denseRankPower =- rarely() ? -1 : (byte) (random().nextInt(7) + 7); // sane + chance of disable-- BitSet set = createSetWithRandomBlocks(BLOCKS);- try (Directory dir = newDirectory()) {- final int cardinality = set.cardinality();- int jumpTableEntryCount;- try (IndexOutput out = EndiannessReverserUtil.createOutput(dir, "foo", IOContext.DEFAULT)) {- jumpTableEntryCount =- IndexedDISI.writeBitSet(new BitSetIterator(set, cardinality), out, denseRankPower);- }- try (IndexInput fullInput = EndiannessReverserUtil.openInput(dir, "foo", IOContext.DEFAULT)) {- IndexInput blockData =- IndexedDISI.createBlockSlice(- fullInput, "blocks", 0, fullInput.length(), jumpTableEntryCount);- blockData.seek(random().nextInt((int) blockData.length()));-- RandomAccessInput jumpTable =- IndexedDISI.createJumpTable(fullInput, 0, fullInput.length(), jumpTableEntryCount);- IndexedDISI disi =- new IndexedDISI(blockData, jumpTable, jumpTableEntryCount, denseRankPower, cardinality);- // This failed at some point during LUCENE-8585 development as it did not reset the slice- // position- disi.advanceExact(BLOCKS * 65536 - 1);- }- }- }-- private BitSet createSetWithRandomBlocks(int blockCount) {- final int B = 65536;- BitSet set = new SparseFixedBitSet(blockCount * B);- for (int block = 0; block < blockCount; block++) {- switch (random().nextInt(4)) {- case 0:- { // EMPTY- break;- }- case 1:- { // ALL- for (int docID = block * B; docID < (block + 1) * B; docID++) {- set.set(docID);- }- break;- }- case 2:- { // SPARSE ( < 4096 )- for (int docID = block * B; docID < (block + 1) * B; docID += 101) {- set.set(docID);- }- break;- }- case 3:- { // DENSE ( >= 4096 )- for (int docID = block * B; docID < (block + 1) * B; docID += 3) {- set.set(docID);- }- break;- }- default:- throw new IllegalStateException(- "Modulo logic error: there should only be 4 possibilities");- }- }- return set;- }-- private void doTestAllSingleJump(BitSet set, Directory dir) throws IOException {- final int cardinality = set.cardinality();- final byte denseRankPower =- rarely() ? -1 : (byte) (random().nextInt(7) + 7); // sane + chance of disable- long length;- int jumpTableentryCount;- try (IndexOutput out = EndiannessReverserUtil.createOutput(dir, "foo", IOContext.DEFAULT)) {- jumpTableentryCount =- IndexedDISI.writeBitSet(new BitSetIterator(set, cardinality), out, denseRankPower);- length = out.getFilePointer();- }-- try (IndexInput in = EndiannessReverserUtil.openInput(dir, "foo", IOContext.DEFAULT)) {- for (int i = 0; i < set.length(); i++) {- IndexedDISI disi =- new IndexedDISI(in, 0L, length, jumpTableentryCount, denseRankPower, cardinality);- assertEquals(- "The bit at " + i + " should be correct with advanceExact",- set.get(i),- disi.advanceExact(i));-- IndexedDISI disi2 =- new IndexedDISI(in, 0L, length, jumpTableentryCount, denseRankPower, cardinality);- disi2.advance(i);- // Proper sanity check with jump tables as an error could make them seek backwards- assertTrue(- "The docID should at least be "- + i- + " after advance("- + i- + ") but was "- + disi2.docID(),- i <= disi2.docID());- if (set.get(i)) {- assertEquals("The docID should be present with advance", i, disi2.docID());- } else {- assertNotSame("The docID should not be present with advance", i, disi2.docID());- }- }- }- }-- public void testOneDoc() throws IOException {- int maxDoc = TestUtil.nextInt(random(), 1, 100000);- BitSet set = new SparseFixedBitSet(maxDoc);- set.set(random().nextInt(maxDoc));- try (Directory dir = newDirectory()) {- doTest(set, dir);- }- }-- public void testTwoDocs() throws IOException {- int maxDoc = TestUtil.nextInt(random(), 1, 100000);- BitSet set = new SparseFixedBitSet(maxDoc);- set.set(random().nextInt(maxDoc));- set.set(random().nextInt(maxDoc));- try (Directory dir = newDirectory()) {- doTest(set, dir);- }- }-- public void testAllDocs() throws IOException {- int maxDoc = TestUtil.nextInt(random(), 1, 100000);- FixedBitSet set = new FixedBitSet(maxDoc);- set.set(1, maxDoc);- try (Directory dir = newDirectory()) {- doTest(set, dir);- }- }-- public void testHalfFull() throws IOException {- int maxDoc = TestUtil.nextInt(random(), 1, 100000);- BitSet set = new SparseFixedBitSet(maxDoc);- for (int i = random().nextInt(2); i < maxDoc; i += TestUtil.nextInt(random(), 1, 3)) {- set.set(i);- }- try (Directory dir = newDirectory()) {- doTest(set, dir);- }- }-- public void testDocRange() throws IOException {- try (Directory dir = newDirectory()) {- for (int iter = 0; iter < 10; ++iter) {- int maxDoc = TestUtil.nextInt(random(), 1, 1000000);- FixedBitSet set = new FixedBitSet(maxDoc);- final int start = random().nextInt(maxDoc);- final int end = TestUtil.nextInt(random(), start + 1, maxDoc);- set.set(start, end);- doTest(set, dir);- }- }- }-- public void testSparseDenseBoundary() throws IOException {- try (Directory dir = newDirectory()) {- FixedBitSet set = new FixedBitSet(200000);- int start = 65536 + random().nextInt(100);- final byte denseRankPower =- rarely() ? -1 : (byte) (random().nextInt(7) + 7); // sane + chance of disable-- // we set MAX_ARRAY_LENGTH bits so the encoding will be sparse- set.set(start, start + IndexedDISI.MAX_ARRAY_LENGTH);- long length;- int jumpTableEntryCount;- try (IndexOutput out =- EndiannessReverserUtil.createOutput(dir, "sparse", IOContext.DEFAULT)) {- jumpTableEntryCount =- IndexedDISI.writeBitSet(- new BitSetIterator(set, IndexedDISI.MAX_ARRAY_LENGTH), out, denseRankPower);- length = out.getFilePointer();- }- try (IndexInput in = EndiannessReverserUtil.openInput(dir, "sparse", IOContext.DEFAULT)) {- IndexedDISI disi =- new IndexedDISI(- in, 0L, length, jumpTableEntryCount, denseRankPower, IndexedDISI.MAX_ARRAY_LENGTH);- assertEquals(start, disi.nextDoc());- assertEquals(IndexedDISI.Method.SPARSE, disi.method);- }- doTest(set, dir);-- // now we set one more bit so the encoding will be dense- set.set(start + IndexedDISI.MAX_ARRAY_LENGTH + random().nextInt(100));- try (IndexOutput out = EndiannessReverserUtil.createOutput(dir, "bar", IOContext.DEFAULT)) {- IndexedDISI.writeBitSet(- new BitSetIterator(set, IndexedDISI.MAX_ARRAY_LENGTH + 1), out, denseRankPower);- length = out.getFilePointer();- }- try (IndexInput in = EndiannessReverserUtil.openInput(dir, "bar", IOContext.DEFAULT)) {- IndexedDISI disi =- new IndexedDISI(- in,- 0L,- length,- jumpTableEntryCount,- denseRankPower,- IndexedDISI.MAX_ARRAY_LENGTH + 1);- assertEquals(start, disi.nextDoc());- assertEquals(IndexedDISI.Method.DENSE, disi.method);- }- doTest(set, dir);- }- }-- public void testOneDocMissing() throws IOException {- int maxDoc = TestUtil.nextInt(random(), 1, 1000000);- FixedBitSet set = new FixedBitSet(maxDoc);- set.set(0, maxDoc);- set.clear(random().nextInt(maxDoc));- try (Directory dir = newDirectory()) {- doTest(set, dir);- }- }-- public void testFewMissingDocs() throws IOException {- try (Directory dir = newDirectory()) {- int numIters = atLeast(10);- for (int iter = 0; iter < numIters; ++iter) {- int maxDoc = TestUtil.nextInt(random(), 1, 100000);- FixedBitSet set = new FixedBitSet(maxDoc);- set.set(0, maxDoc);- final int numMissingDocs = TestUtil.nextInt(random(), 2, 1000);- for (int i = 0; i < numMissingDocs; ++i) {- set.clear(random().nextInt(maxDoc));- }- doTest(set, dir);- }- }- }-- public void testDenseMultiBlock() throws IOException {- try (Directory dir = newDirectory()) {- int maxDoc = 10 * 65536; // 10 blocks- FixedBitSet set = new FixedBitSet(maxDoc);- for (int i = 0; i < maxDoc; i += 2) { // Set every other to ensure dense- set.set(i);- }- doTest(set, dir);- }- }-- public void testIllegalDenseRankPower() throws IOException {-- // Legal values- for (byte denseRankPower : new byte[] {-1, 7, 8, 9, 10, 11, 12, 13, 14, 15}) {- createAndOpenDISI(denseRankPower, denseRankPower);- }-- // Illegal values- for (byte denseRankPower : new byte[] {-2, 0, 1, 6, 16}) {- expectThrows(- IllegalArgumentException.class,- () -> {- createAndOpenDISI(- denseRankPower, (byte) 8); // Illegal write, legal read (should not reach read)- });-- expectThrows(- IllegalArgumentException.class,- () -> {- createAndOpenDISI(- (byte) 8, denseRankPower); // Legal write, illegal read (should reach read)- });- }- }-- private void createAndOpenDISI(byte denseRankPowerWrite, byte denseRankPowerRead)- throws IOException {- BitSet set = new FixedBitSet(10);- set.set(set.length() - 1);- try (Directory dir = newDirectory()) {- long length;- int jumpTableEntryCount = -1;- try (IndexOutput out = EndiannessReverserUtil.createOutput(dir, "foo", IOContext.DEFAULT)) {- jumpTableEntryCount =- IndexedDISI.writeBitSet(- new BitSetIterator(set, set.cardinality()), out, denseRankPowerWrite);- length = out.getFilePointer();- }- try (IndexInput in = EndiannessReverserUtil.openInput(dir, "foo", IOContext.DEFAULT)) {- new IndexedDISI(in, 0L, length, jumpTableEntryCount, denseRankPowerRead, set.cardinality());- }- // This tests the legality of the denseRankPower only, so we don't do anything with the disi- }- }-- public void testOneDocMissingFixed() throws IOException {- int maxDoc = 9699;- final byte denseRankPower =- rarely() ? -1 : (byte) (random().nextInt(7) + 7); // sane + chance of disable- FixedBitSet set = new FixedBitSet(maxDoc);- set.set(0, maxDoc);- set.clear(1345);- try (Directory dir = newDirectory()) {-- final int cardinality = set.cardinality();- long length;- int jumpTableentryCount;- try (IndexOutput out = EndiannessReverserUtil.createOutput(dir, "foo", IOContext.DEFAULT)) {- jumpTableentryCount =- IndexedDISI.writeBitSet(new BitSetIterator(set, cardinality), out, denseRankPower);- length = out.getFilePointer();- }-- int step = 16000;- try (IndexInput in = EndiannessReverserUtil.openInput(dir, "foo", IOContext.DEFAULT)) {- IndexedDISI disi =- new IndexedDISI(in, 0L, length, jumpTableentryCount, denseRankPower, cardinality);- BitSetIterator disi2 = new BitSetIterator(set, cardinality);- assertAdvanceEquality(disi, disi2, step);- }- }- }-- public void testRandom() throws IOException {- try (Directory dir = newDirectory()) {- int numIters = atLeast(3);- for (int i = 0; i < numIters; ++i) {- doTestRandom(dir);- }- }- }-- private void doTestRandom(Directory dir) throws IOException {- Random random = random();- final int maxStep = TestUtil.nextInt(random, 1, 1 << TestUtil.nextInt(random, 2, 20));- final int numDocs =- TestUtil.nextInt(random, 1, Math.min(100000, (Integer.MAX_VALUE - 1) / maxStep));- BitSet docs = new SparseFixedBitSet(numDocs * maxStep + 1);- int lastDoc = -1;- for (int doc = -1, i = 0; i < numDocs; ++i) {- doc += TestUtil.nextInt(random, 1, maxStep);- docs.set(doc);- lastDoc = doc;- }- final int maxDoc = lastDoc + TestUtil.nextInt(random, 1, 100);-- BitSet set = BitSet.of(new BitSetIterator(docs, docs.approximateCardinality()), maxDoc);- doTest(set, dir);- }-- private void doTest(BitSet set, Directory dir) throws IOException {- final int cardinality = set.cardinality();- final byte denseRankPower =- rarely() ? -1 : (byte) (random().nextInt(7) + 7); // sane + chance of disable- long length;- int jumpTableentryCount;- try (IndexOutput out = EndiannessReverserUtil.createOutput(dir, "foo", IOContext.DEFAULT)) {- jumpTableentryCount =- IndexedDISI.writeBitSet(new BitSetIterator(set, cardinality), out, denseRankPower);- length = out.getFilePointer();- }-- try (IndexInput in = EndiannessReverserUtil.openInput(dir, "foo", IOContext.DEFAULT)) {- IndexedDISI disi =- new IndexedDISI(in, 0L, length, jumpTableentryCount, denseRankPower, cardinality);- BitSetIterator disi2 = new BitSetIterator(set, cardinality);- assertSingleStepEquality(disi, disi2);- }-- for (int step : new int[] {1, 10, 100, 1000, 10000, 100000}) {- try (IndexInput in = EndiannessReverserUtil.openInput(dir, "foo", IOContext.DEFAULT)) {- IndexedDISI disi =- new IndexedDISI(in, 0L, length, jumpTableentryCount, denseRankPower, cardinality);- BitSetIterator disi2 = new BitSetIterator(set, cardinality);- assertAdvanceEquality(disi, disi2, step);- }- }-- for (int step : new int[] {10, 100, 1000, 10000, 100000}) {- try (IndexInput in = EndiannessReverserUtil.openInput(dir, "foo", IOContext.DEFAULT)) {- IndexedDISI disi =- new IndexedDISI(in, 0L, length, jumpTableentryCount, denseRankPower, cardinality);- BitSetIterator disi2 = new BitSetIterator(set, cardinality);- int disi2length = set.length();- assertAdvanceExactRandomized(disi, disi2, disi2length, step);- }- }-- dir.deleteFile("foo");- }-- private void assertAdvanceExactRandomized(- IndexedDISI disi, BitSetIterator disi2, int disi2length, int step) throws IOException {- int index = -1;- Random random = random();- for (int target = 0; target < disi2length; ) {- target += TestUtil.nextInt(random, 0, step);- int doc = disi2.docID();- while (doc < target) {- doc = disi2.nextDoc();- index++;- }-- boolean exists = disi.advanceExact(target);- assertEquals(doc == target, exists);- if (exists) {- assertEquals(index, disi.index());- } else if (random.nextBoolean()) {- assertEquals(doc, disi.nextDoc());- // This is a bit strange when doc == NO_MORE_DOCS as the index overcounts in the disi2- // while-loop- assertEquals(index, disi.index());- target = doc;- }- }- }-- private void assertSingleStepEquality(IndexedDISI disi, BitSetIterator disi2) throws IOException {- int i = 0;- for (int doc = disi2.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = disi2.nextDoc()) {- assertEquals(doc, disi.nextDoc());- assertEquals(i++, disi.index());- }- assertEquals(DocIdSetIterator.NO_MORE_DOCS, disi.nextDoc());- }-- private void assertAdvanceEquality(IndexedDISI disi, BitSetIterator disi2, int step)- throws IOException {- int index = -1;- while (true) {- int target = disi2.docID() + step;- int doc;- do {- doc = disi2.nextDoc();- index++;- } while (doc < target);- assertEquals(doc, disi.advance(target));- if (doc == DocIdSetIterator.NO_MORE_DOCS) {- break;- }- assertEquals(- "Expected equality using step " + step + " at docID " + doc, index, disi.index());- }- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestLucene80NormsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestLucene80NormsFormat.javadeleted file mode 100644index c61fb2beae4..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestLucene80NormsFormat.java+++ /dev/null@@ -1,33 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene80;--import org.apache.lucene.backward_codecs.lucene87.Lucene87RWCodec;-import org.apache.lucene.codecs.Codec;-import org.apache.lucene.tests.index.BaseNormsFormatTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;--/** Tests Lucene80NormsFormat */-@Nightly // N-2 formats are only tested on nightly runs-public class TestLucene80NormsFormat extends BaseNormsFormatTestCase {- private final Codec codec = new Lucene87RWCodec();-- @Override- protected Codec getCodec() {- return codec;- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestLucene80NormsFormatMergeInstance.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestLucene80NormsFormatMergeInstance.javadeleted file mode 100644index 851d8f82056..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestLucene80NormsFormatMergeInstance.java+++ /dev/null@@ -1,29 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene80;--import org.apache.lucene.tests.util.LuceneTestCase.Nightly;--/** Test the merge instance of the Lucene80 norms format. */-@Nightly // N-2 formats are only tested on nightly runs-public class TestLucene80NormsFormatMergeInstance extends TestLucene80NormsFormat {-- @Override- protected boolean shouldTestMergeInstance() {- return true;- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84PostingsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84PostingsWriter.javadeleted file mode 100644index 3fa503c033e..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84PostingsWriter.java+++ /dev/null@@ -1,549 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene84;--import static org.apache.lucene.backward_codecs.lucene84.ForUtil.BLOCK_SIZE;-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.DOC_CODEC;-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.MAX_SKIP_LEVELS;-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.PAY_CODEC;-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.POS_CODEC;-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.TERMS_CODEC;-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.VERSION_CURRENT;--import java.io.IOException;-import java.nio.ByteOrder;-import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.IntBlockTermState;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.BlockTermState;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.CompetitiveImpactAccumulator;-import org.apache.lucene.codecs.PushPostingsWriterBase;-import org.apache.lucene.index.CorruptIndexException;-import org.apache.lucene.index.FieldInfo;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.IndexWriter;-import org.apache.lucene.index.NumericDocValues;-import org.apache.lucene.index.SegmentWriteState;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.store.IndexOutput;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.BitUtil;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.IOUtils;--/**- * Concrete class that writes docId(maybe frq,pos,offset,payloads) list with postings format.- *- * <p>Postings list for each term will be stored separately.- *- * @see Lucene84SkipWriter for details about skipping setting and postings layout.- * @lucene.experimental- */-public final class Lucene84PostingsWriter extends PushPostingsWriterBase {-- IndexOutput docOut;- IndexOutput posOut;- IndexOutput payOut;-- static final IntBlockTermState emptyState = new IntBlockTermState();- IntBlockTermState lastState;-- // Holds starting file pointers for current term:- private long docStartFP;- private long posStartFP;- private long payStartFP;-- final long[] docDeltaBuffer;- final long[] freqBuffer;- private int docBufferUpto;-- final long[] posDeltaBuffer;- final long[] payloadLengthBuffer;- final long[] offsetStartDeltaBuffer;- final long[] offsetLengthBuffer;- private int posBufferUpto;-- private byte[] payloadBytes;- private int payloadByteUpto;-- private int lastBlockDocID;- private long lastBlockPosFP;- private long lastBlockPayFP;- private int lastBlockPosBufferUpto;- private int lastBlockPayloadByteUpto;-- private int lastDocID;- private int lastPosition;- private int lastStartOffset;- private int docCount;-- private final PForUtil pforUtil;- private final ForDeltaUtil forDeltaUtil;- private final Lucene84SkipWriter skipWriter;-- private boolean fieldHasNorms;- private NumericDocValues norms;- private final CompetitiveImpactAccumulator competitiveFreqNormAccumulator =- new CompetitiveImpactAccumulator();-- /** Creates a postings writer */- public Lucene84PostingsWriter(SegmentWriteState state) throws IOException {-- String docFileName =- IndexFileNames.segmentFileName(- state.segmentInfo.name, state.segmentSuffix, Lucene84PostingsFormat.DOC_EXTENSION);- docOut = EndiannessReverserUtil.createOutput(state.directory, docFileName, state.context);- IndexOutput posOut = null;- IndexOutput payOut = null;- boolean success = false;- try {- CodecUtil.writeIndexHeader(- docOut, DOC_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);- ByteOrder byteOrder = ByteOrder.nativeOrder();- if (byteOrder == ByteOrder.BIG_ENDIAN) {- docOut.writeByte((byte) 'B');- } else if (byteOrder == ByteOrder.LITTLE_ENDIAN) {- docOut.writeByte((byte) 'L');- } else {- throw new Error();- }- final ForUtil forUtil = new ForUtil();- forDeltaUtil = new ForDeltaUtil(forUtil);- pforUtil = new PForUtil(forUtil);- if (state.fieldInfos.hasProx()) {- posDeltaBuffer = new long[BLOCK_SIZE];- String posFileName =- IndexFileNames.segmentFileName(- state.segmentInfo.name, state.segmentSuffix, Lucene84PostingsFormat.POS_EXTENSION);- posOut = EndiannessReverserUtil.createOutput(state.directory, posFileName, state.context);- CodecUtil.writeIndexHeader(- posOut, POS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);-- if (state.fieldInfos.hasPayloads()) {- payloadBytes = new byte[128];- payloadLengthBuffer = new long[BLOCK_SIZE];- } else {- payloadBytes = null;- payloadLengthBuffer = null;- }-- if (state.fieldInfos.hasOffsets()) {- offsetStartDeltaBuffer = new long[BLOCK_SIZE];- offsetLengthBuffer = new long[BLOCK_SIZE];- } else {- offsetStartDeltaBuffer = null;- offsetLengthBuffer = null;- }-- if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {- String payFileName =- IndexFileNames.segmentFileName(- state.segmentInfo.name,- state.segmentSuffix,- Lucene84PostingsFormat.PAY_EXTENSION);- payOut = EndiannessReverserUtil.createOutput(state.directory, payFileName, state.context);- CodecUtil.writeIndexHeader(- payOut, PAY_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);- }- } else {- posDeltaBuffer = null;- payloadLengthBuffer = null;- offsetStartDeltaBuffer = null;- offsetLengthBuffer = null;- payloadBytes = null;- }- this.payOut = payOut;- this.posOut = posOut;- success = true;- } finally {- if (!success) {- IOUtils.closeWhileHandlingException(docOut, posOut, payOut);- }- }-- docDeltaBuffer = new long[BLOCK_SIZE];- freqBuffer = new long[BLOCK_SIZE];-- // TODO: should we try skipping every 2/4 blocks...?- skipWriter =- new Lucene84SkipWriter(- MAX_SKIP_LEVELS, BLOCK_SIZE, state.segmentInfo.maxDoc(), docOut, posOut, payOut);- }-- @Override- public IntBlockTermState newTermState() {- return new IntBlockTermState();- }-- @Override- public void init(IndexOutput termsOut, SegmentWriteState state) throws IOException {- CodecUtil.writeIndexHeader(- termsOut, TERMS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);- termsOut.writeVInt(BLOCK_SIZE);- }-- @Override- public void setField(FieldInfo fieldInfo) {- super.setField(fieldInfo);- skipWriter.setField(writePositions, writeOffsets, writePayloads);- lastState = emptyState;- fieldHasNorms = fieldInfo.hasNorms();- }-- @Override- public void startTerm(NumericDocValues norms) {- docStartFP = docOut.getFilePointer();- if (writePositions) {- posStartFP = posOut.getFilePointer();- if (writePayloads || writeOffsets) {- payStartFP = payOut.getFilePointer();- }- }- lastDocID = 0;- lastBlockDocID = -1;- skipWriter.resetSkip();- this.norms = norms;- competitiveFreqNormAccumulator.clear();- }-- @Override- public void startDoc(int docID, int termDocFreq) throws IOException {- // Have collected a block of docs, and get a new doc.- // Should write skip data as well as postings list for- // current block.- if (lastBlockDocID != -1 && docBufferUpto == 0) {- skipWriter.bufferSkip(- lastBlockDocID,- competitiveFreqNormAccumulator,- docCount,- lastBlockPosFP,- lastBlockPayFP,- lastBlockPosBufferUpto,- lastBlockPayloadByteUpto);- competitiveFreqNormAccumulator.clear();- }-- final int docDelta = docID - lastDocID;-- if (docID < 0 || (docCount > 0 && docDelta <= 0)) {- throw new CorruptIndexException(- "docs out of order (" + docID + " <= " + lastDocID + " )", docOut);- }-- docDeltaBuffer[docBufferUpto] = docDelta;- if (writeFreqs) {- freqBuffer[docBufferUpto] = termDocFreq;- }-- docBufferUpto++;- docCount++;-- if (docBufferUpto == BLOCK_SIZE) {- forDeltaUtil.encodeDeltas(docDeltaBuffer, docOut);- if (writeFreqs) {- pforUtil.encode(freqBuffer, docOut);- }- // NOTE: don't set docBufferUpto back to 0 here;- // finishDoc will do so (because it needs to see that- // the block was filled so it can save skip data)- }-- lastDocID = docID;- lastPosition = 0;- lastStartOffset = 0;-- long norm;- if (fieldHasNorms) {- boolean found = norms.advanceExact(docID);- if (found == false) {- // This can happen if indexing hits a problem after adding a doc to the- // postings but before buffering the norm. Such documents are written- // deleted and will go away on the first merge.- norm = 1L;- } else {- norm = norms.longValue();- assert norm != 0 : docID;- }- } else {- norm = 1L;- }-- competitiveFreqNormAccumulator.add(writeFreqs ? termDocFreq : 1, norm);- }-- @Override- public void addPosition(int position, BytesRef payload, int startOffset, int endOffset)- throws IOException {- if (position > IndexWriter.MAX_POSITION) {- throw new CorruptIndexException(- "position="- + position- + " is too large (> IndexWriter.MAX_POSITION="- + IndexWriter.MAX_POSITION- + ")",- docOut);- }- if (position < 0) {- throw new CorruptIndexException("position=" + position + " is < 0", docOut);- }- posDeltaBuffer[posBufferUpto] = position - lastPosition;- if (writePayloads) {- if (payload == null || payload.length == 0) {- // no payload- payloadLengthBuffer[posBufferUpto] = 0;- } else {- payloadLengthBuffer[posBufferUpto] = payload.length;- if (payloadByteUpto + payload.length > payloadBytes.length) {- payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payload.length);- }- System.arraycopy(- payload.bytes, payload.offset, payloadBytes, payloadByteUpto, payload.length);- payloadByteUpto += payload.length;- }- }-- if (writeOffsets) {- assert startOffset >= lastStartOffset;- assert endOffset >= startOffset;- offsetStartDeltaBuffer[posBufferUpto] = startOffset - lastStartOffset;- offsetLengthBuffer[posBufferUpto] = endOffset - startOffset;- lastStartOffset = startOffset;- }-- posBufferUpto++;- lastPosition = position;- if (posBufferUpto == BLOCK_SIZE) {- pforUtil.encode(posDeltaBuffer, posOut);-- if (writePayloads) {- pforUtil.encode(payloadLengthBuffer, payOut);- payOut.writeVInt(payloadByteUpto);- payOut.writeBytes(payloadBytes, 0, payloadByteUpto);- payloadByteUpto = 0;- }- if (writeOffsets) {- pforUtil.encode(offsetStartDeltaBuffer, payOut);- pforUtil.encode(offsetLengthBuffer, payOut);- }- posBufferUpto = 0;- }- }-- @Override- public void finishDoc() throws IOException {- // Since we don't know df for current term, we had to buffer- // those skip data for each block, and when a new doc comes,- // write them to skip file.- if (docBufferUpto == BLOCK_SIZE) {- lastBlockDocID = lastDocID;- if (posOut != null) {- if (payOut != null) {- lastBlockPayFP = payOut.getFilePointer();- }- lastBlockPosFP = posOut.getFilePointer();- lastBlockPosBufferUpto = posBufferUpto;- lastBlockPayloadByteUpto = payloadByteUpto;- }- docBufferUpto = 0;- }- }-- /** Called when we are done adding docs to this term */- @Override- public void finishTerm(BlockTermState _state) throws IOException {- IntBlockTermState state = (IntBlockTermState) _state;- assert state.docFreq > 0;-- // TODO: wasteful we are counting this (counting # docs- // for this term) in two places?- assert state.docFreq == docCount : state.docFreq + " vs " + docCount;-- // docFreq == 1, don't write the single docid/freq to a separate file along with a pointer to- // it.- final int singletonDocID;- if (state.docFreq == 1) {- // pulse the singleton docid into the term dictionary, freq is implicitly totalTermFreq- singletonDocID = (int) docDeltaBuffer[0];- } else {- singletonDocID = -1;- // vInt encode the remaining doc deltas and freqs:- for (int i = 0; i < docBufferUpto; i++) {- final int docDelta = (int) docDeltaBuffer[i];- final int freq = (int) freqBuffer[i];- if (!writeFreqs) {- docOut.writeVInt(docDelta);- } else if (freq == 1) {- docOut.writeVInt((docDelta << 1) | 1);- } else {- docOut.writeVInt(docDelta << 1);- docOut.writeVInt(freq);- }- }- }-- final long lastPosBlockOffset;-- if (writePositions) {- // totalTermFreq is just total number of positions(or payloads, or offsets)- // associated with current term.- assert state.totalTermFreq != -1;- if (state.totalTermFreq > BLOCK_SIZE) {- // record file offset for last pos in last block- lastPosBlockOffset = posOut.getFilePointer() - posStartFP;- } else {- lastPosBlockOffset = -1;- }- if (posBufferUpto > 0) {- // TODO: should we send offsets/payloads to- // .pay...? seems wasteful (have to store extra- // vLong for low (< BLOCK_SIZE) DF terms = vast vast- // majority)-- // vInt encode the remaining positions/payloads/offsets:- int lastPayloadLength = -1; // force first payload length to be written- int lastOffsetLength = -1; // force first offset length to be written- int payloadBytesReadUpto = 0;- for (int i = 0; i < posBufferUpto; i++) {- final int posDelta = (int) posDeltaBuffer[i];- if (writePayloads) {- final int payloadLength = (int) payloadLengthBuffer[i];- if (payloadLength != lastPayloadLength) {- lastPayloadLength = payloadLength;- posOut.writeVInt((posDelta << 1) | 1);- posOut.writeVInt(payloadLength);- } else {- posOut.writeVInt(posDelta << 1);- }-- if (payloadLength != 0) {- posOut.writeBytes(payloadBytes, payloadBytesReadUpto, payloadLength);- payloadBytesReadUpto += payloadLength;- }- } else {- posOut.writeVInt(posDelta);- }-- if (writeOffsets) {- int delta = (int) offsetStartDeltaBuffer[i];- int length = (int) offsetLengthBuffer[i];- if (length == lastOffsetLength) {- posOut.writeVInt(delta << 1);- } else {- posOut.writeVInt(delta << 1 | 1);- posOut.writeVInt(length);- lastOffsetLength = length;- }- }- }-- if (writePayloads) {- assert payloadBytesReadUpto == payloadByteUpto;- payloadByteUpto = 0;- }- }- } else {- lastPosBlockOffset = -1;- }-- long skipOffset;- if (docCount > BLOCK_SIZE) {- skipOffset = skipWriter.writeSkip(docOut) - docStartFP;- } else {- skipOffset = -1;- }-- state.docStartFP = docStartFP;- state.posStartFP = posStartFP;- state.payStartFP = payStartFP;- state.singletonDocID = singletonDocID;- state.skipOffset = skipOffset;- state.lastPosBlockOffset = lastPosBlockOffset;- docBufferUpto = 0;- posBufferUpto = 0;- lastDocID = 0;- docCount = 0;- }-- @Override- public void encodeTerm(- DataOutput out, FieldInfo fieldInfo, BlockTermState _state, boolean absolute)- throws IOException {- IntBlockTermState state = (IntBlockTermState) _state;- if (absolute) {- lastState = emptyState;- assert lastState.docStartFP == 0;- }-- if (lastState.singletonDocID != -1- && state.singletonDocID != -1- && state.docStartFP == lastState.docStartFP) {- // With runs of rare values such as ID fields, the increment of pointers in the docs file is- // often 0.- // Furthermore some ID schemes like auto-increment IDs or Flake IDs are monotonic, so we- // encode the delta- // between consecutive doc IDs to save space.- final long delta = (long) state.singletonDocID - lastState.singletonDocID;- out.writeVLong((BitUtil.zigZagEncode(delta) << 1) | 0x01);- } else {- out.writeVLong((state.docStartFP - lastState.docStartFP) << 1);- if (state.singletonDocID != -1) {- out.writeVInt(state.singletonDocID);- }- }-- if (writePositions) {- out.writeVLong(state.posStartFP - lastState.posStartFP);- if (writePayloads || writeOffsets) {- out.writeVLong(state.payStartFP - lastState.payStartFP);- }- }- if (writePositions) {- if (state.lastPosBlockOffset != -1) {- out.writeVLong(state.lastPosBlockOffset);- }- }- if (state.skipOffset != -1) {- out.writeVLong(state.skipOffset);- }- lastState = state;- }-- @Override- public void close() throws IOException {- // TODO: add a finish() at least to PushBase? DV too...?- boolean success = false;- try {- if (docOut != null) {- CodecUtil.writeFooter(docOut);- }- if (posOut != null) {- CodecUtil.writeFooter(posOut);- }- if (payOut != null) {- CodecUtil.writeFooter(payOut);- }- success = true;- } finally {- if (success) {- IOUtils.close(docOut, posOut, payOut);- } else {- IOUtils.closeWhileHandlingException(docOut, posOut, payOut);- }- docOut = posOut = payOut = null;- }- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84RWCodec.javadeleted file mode 100644index b129f5a2ac6..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84RWCodec.java+++ /dev/null@@ -1,81 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene84;--import org.apache.lucene.backward_codecs.lucene50.Lucene50RWCompoundFormat;-import org.apache.lucene.backward_codecs.lucene50.Lucene50RWStoredFieldsFormat;-import org.apache.lucene.backward_codecs.lucene50.Lucene50RWTermVectorsFormat;-import org.apache.lucene.backward_codecs.lucene60.Lucene60RWPointsFormat;-import org.apache.lucene.backward_codecs.lucene70.Lucene70RWSegmentInfoFormat;-import org.apache.lucene.backward_codecs.lucene80.Lucene80RWNormsFormat;-import org.apache.lucene.codecs.CompoundFormat;-import org.apache.lucene.codecs.NormsFormat;-import org.apache.lucene.codecs.PointsFormat;-import org.apache.lucene.codecs.PostingsFormat;-import org.apache.lucene.codecs.SegmentInfoFormat;-import org.apache.lucene.codecs.StoredFieldsFormat;-import org.apache.lucene.codecs.TermVectorsFormat;-import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;--/** RW impersonation of {@link Lucene84Codec}. */-public class Lucene84RWCodec extends Lucene84Codec {-- private final PostingsFormat defaultPF = new Lucene84RWPostingsFormat();- private final CompoundFormat compoundFormat = new Lucene50RWCompoundFormat();- private final PostingsFormat postingsFormat =- new PerFieldPostingsFormat() {- @Override- public PostingsFormat getPostingsFormatForField(String field) {- return defaultPF;- }- };-- @Override- public PointsFormat pointsFormat() {- return new Lucene60RWPointsFormat();- }-- @Override- public SegmentInfoFormat segmentInfoFormat() {- return new Lucene70RWSegmentInfoFormat();- }-- @Override- public StoredFieldsFormat storedFieldsFormat() {- return new Lucene50RWStoredFieldsFormat();- }-- @Override- public NormsFormat normsFormat() {- return new Lucene80RWNormsFormat();- }-- @Override- public PostingsFormat postingsFormat() {- return postingsFormat;- }-- @Override- public final CompoundFormat compoundFormat() {- return compoundFormat;- }-- @Override- public TermVectorsFormat termVectorsFormat() {- return new Lucene50RWTermVectorsFormat();- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84RWPostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84RWPostingsFormat.javadeleted file mode 100644index 0b88d6e7dcf..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84RWPostingsFormat.java+++ /dev/null@@ -1,46 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene84;--import java.io.IOException;-import org.apache.lucene.backward_codecs.lucene40.blocktree.Lucene40BlockTreeTermsWriter;-import org.apache.lucene.codecs.FieldsConsumer;-import org.apache.lucene.codecs.PostingsWriterBase;-import org.apache.lucene.index.SegmentWriteState;-import org.apache.lucene.util.IOUtils;--public class Lucene84RWPostingsFormat extends Lucene84PostingsFormat {- @Override- public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {- PostingsWriterBase postingsWriter = new Lucene84PostingsWriter(state);- boolean success = false;- try {- FieldsConsumer ret =- new Lucene40BlockTreeTermsWriter(- state,- postingsWriter,- Lucene40BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,- Lucene40BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);- success = true;- return ret;- } finally {- if (!success) {- IOUtils.closeWhileHandlingException(postingsWriter);- }- }- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84SkipWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84SkipWriter.javadeleted file mode 100644index 3d063f8c799..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84SkipWriter.java+++ /dev/null@@ -1,237 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene84;--import java.io.IOException;-import java.util.Arrays;-import java.util.Collection;-import org.apache.lucene.codecs.CompetitiveImpactAccumulator;-import org.apache.lucene.codecs.Impact;-import org.apache.lucene.codecs.MultiLevelSkipListWriter;-import org.apache.lucene.store.ByteBuffersDataOutput;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.store.IndexOutput;--/**- * Write skip lists with multiple levels, and support skip within block ints.- *- * <p>Assume that docFreq = 28, skipInterval = blockSize = 12- *- * <pre>- * | block#0 | | block#1 | |vInts|- * d d d d d d d d d d d d d d d d d d d d d d d d d d d d (posting list)- * ^ ^ (level 0 skip point)- * </pre>- *- * <p>Note that skipWriter will ignore first document in block#0, since it is useless as a skip- * point. Also, we'll never skip into the vInts block, only record skip data at the start its start- * point(if it exist).- *- * <p>For each skip point, we will record: 1. docID in former position, i.e. for position 12, record- * docID[11], etc. 2. its related file points(position, payload), 3. related numbers or- * uptos(position, payload). 4. start offset.- */-final class Lucene84SkipWriter extends MultiLevelSkipListWriter {- private int[] lastSkipDoc;- private long[] lastSkipDocPointer;- private long[] lastSkipPosPointer;- private long[] lastSkipPayPointer;- private int[] lastPayloadByteUpto;-- private final IndexOutput docOut;- private final IndexOutput posOut;- private final IndexOutput payOut;-- private int curDoc;- private long curDocPointer;- private long curPosPointer;- private long curPayPointer;- private int curPosBufferUpto;- private int curPayloadByteUpto;- private CompetitiveImpactAccumulator[] curCompetitiveFreqNorms;- private boolean fieldHasPositions;- private boolean fieldHasOffsets;- private boolean fieldHasPayloads;-- public Lucene84SkipWriter(- int maxSkipLevels,- int blockSize,- int docCount,- IndexOutput docOut,- IndexOutput posOut,- IndexOutput payOut) {- super(blockSize, 8, maxSkipLevels, docCount);- this.docOut = docOut;- this.posOut = posOut;- this.payOut = payOut;-- lastSkipDoc = new int[maxSkipLevels];- lastSkipDocPointer = new long[maxSkipLevels];- if (posOut != null) {- lastSkipPosPointer = new long[maxSkipLevels];- if (payOut != null) {- lastSkipPayPointer = new long[maxSkipLevels];- }- lastPayloadByteUpto = new int[maxSkipLevels];- }- curCompetitiveFreqNorms = new CompetitiveImpactAccumulator[maxSkipLevels];- for (int i = 0; i < maxSkipLevels; ++i) {- curCompetitiveFreqNorms[i] = new CompetitiveImpactAccumulator();- }- }-- public void setField(- boolean fieldHasPositions, boolean fieldHasOffsets, boolean fieldHasPayloads) {- this.fieldHasPositions = fieldHasPositions;- this.fieldHasOffsets = fieldHasOffsets;- this.fieldHasPayloads = fieldHasPayloads;- }-- // tricky: we only skip data for blocks (terms with more than 128 docs), but re-init'ing the- // skipper- // is pretty slow for rare terms in large segments as we have to fill O(log #docs in segment) of- // junk.- // this is the vast majority of terms (worst case: ID field or similar). so in resetSkip() we- // save- // away the previous pointers, and lazy-init only if we need to buffer skip data for the term.- private boolean initialized;- long lastDocFP;- long lastPosFP;- long lastPayFP;-- @Override- public void resetSkip() {- lastDocFP = docOut.getFilePointer();- if (fieldHasPositions) {- lastPosFP = posOut.getFilePointer();- if (fieldHasOffsets || fieldHasPayloads) {- lastPayFP = payOut.getFilePointer();- }- }- if (initialized) {- for (CompetitiveImpactAccumulator acc : curCompetitiveFreqNorms) {- acc.clear();- }- }- initialized = false;- }-- private void initSkip() {- if (!initialized) {- super.resetSkip();- Arrays.fill(lastSkipDoc, 0);- Arrays.fill(lastSkipDocPointer, lastDocFP);- if (fieldHasPositions) {- Arrays.fill(lastSkipPosPointer, lastPosFP);- if (fieldHasPayloads) {- Arrays.fill(lastPayloadByteUpto, 0);- }- if (fieldHasOffsets || fieldHasPayloads) {- Arrays.fill(lastSkipPayPointer, lastPayFP);- }- }- // sets of competitive freq,norm pairs should be empty at this point- assert Arrays.stream(curCompetitiveFreqNorms)- .map(CompetitiveImpactAccumulator::getCompetitiveFreqNormPairs)- .mapToInt(Collection::size)- .sum()- == 0;- initialized = true;- }- }-- /** Sets the values for the current skip data. */- public void bufferSkip(- int doc,- CompetitiveImpactAccumulator competitiveFreqNorms,- int numDocs,- long posFP,- long payFP,- int posBufferUpto,- int payloadByteUpto)- throws IOException {- initSkip();- this.curDoc = doc;- this.curDocPointer = docOut.getFilePointer();- this.curPosPointer = posFP;- this.curPayPointer = payFP;- this.curPosBufferUpto = posBufferUpto;- this.curPayloadByteUpto = payloadByteUpto;- this.curCompetitiveFreqNorms[0].addAll(competitiveFreqNorms);- bufferSkip(numDocs);- }-- private final ByteBuffersDataOutput freqNormOut = ByteBuffersDataOutput.newResettableInstance();-- @Override- protected void writeSkipData(int level, DataOutput skipBuffer) throws IOException {-- int delta = curDoc - lastSkipDoc[level];-- skipBuffer.writeVInt(delta);- lastSkipDoc[level] = curDoc;-- skipBuffer.writeVLong(curDocPointer - lastSkipDocPointer[level]);- lastSkipDocPointer[level] = curDocPointer;-- if (fieldHasPositions) {-- skipBuffer.writeVLong(curPosPointer - lastSkipPosPointer[level]);- lastSkipPosPointer[level] = curPosPointer;- skipBuffer.writeVInt(curPosBufferUpto);-- if (fieldHasPayloads) {- skipBuffer.writeVInt(curPayloadByteUpto);- }-- if (fieldHasOffsets || fieldHasPayloads) {- skipBuffer.writeVLong(curPayPointer - lastSkipPayPointer[level]);- lastSkipPayPointer[level] = curPayPointer;- }- }-- CompetitiveImpactAccumulator competitiveFreqNorms = curCompetitiveFreqNorms[level];- assert competitiveFreqNorms.getCompetitiveFreqNormPairs().size() > 0;- if (level + 1 < numberOfSkipLevels) {- curCompetitiveFreqNorms[level + 1].addAll(competitiveFreqNorms);- }- writeImpacts(competitiveFreqNorms, freqNormOut);- skipBuffer.writeVInt(Math.toIntExact(freqNormOut.size()));- freqNormOut.copyTo(skipBuffer);- freqNormOut.reset();- competitiveFreqNorms.clear();- }-- static void writeImpacts(CompetitiveImpactAccumulator acc, DataOutput out) throws IOException {- Collection<Impact> impacts = acc.getCompetitiveFreqNormPairs();- Impact previous = new Impact(0, 0);- for (Impact impact : impacts) {- assert impact.freq > previous.freq;- assert Long.compareUnsigned(impact.norm, previous.norm) > 0;- int freqDelta = impact.freq - previous.freq - 1;- long normDelta = impact.norm - previous.norm - 1;- if (normDelta == 0) {- // most of time, norm only increases by 1, so we can fold everything in a single byte- out.writeVInt(freqDelta << 1);- } else {- out.writeVInt((freqDelta << 1) | 1);- out.writeZLong(normDelta);- }- previous = impact;- }- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestForDeltaUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestForDeltaUtil.javadeleted file mode 100644index 8217751151a..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestForDeltaUtil.java+++ /dev/null@@ -1,96 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene84;--import com.carrotsearch.randomizedtesting.generators.RandomNumbers;-import java.io.IOException;-import java.util.Arrays;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.store.ByteBuffersDirectory;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.store.IndexOutput;-import org.apache.lucene.tests.util.LuceneTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;-import org.apache.lucene.tests.util.TestUtil;-import org.apache.lucene.util.packed.PackedInts;--@Nightly // N-2 formats are only tested on nightly runs-public class TestForDeltaUtil extends LuceneTestCase {-- public void testEncodeDecode() throws IOException {- final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);- final int[] values = new int[iterations * ForUtil.BLOCK_SIZE];-- for (int i = 0; i < iterations; ++i) {- final int bpv = TestUtil.nextInt(random(), 1, 31 - 7);- for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {- values[i * ForUtil.BLOCK_SIZE + j] =- RandomNumbers.randomIntBetween(random(), 1, (int) PackedInts.maxValue(bpv));- }- }-- final Directory d = new ByteBuffersDirectory();- final long endPointer;-- {- // encode- IndexOutput out = EndiannessReverserUtil.createOutput(d, "test.bin", IOContext.DEFAULT);- final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(new ForUtil());-- for (int i = 0; i < iterations; ++i) {- long[] source = new long[ForUtil.BLOCK_SIZE];- for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {- source[j] = values[i * ForUtil.BLOCK_SIZE + j];- }- forDeltaUtil.encodeDeltas(source, out);- }- endPointer = out.getFilePointer();- out.close();- }-- {- // decode- IndexInput in = EndiannessReverserUtil.openInput(d, "test.bin", IOContext.READONCE);- final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(new ForUtil());- for (int i = 0; i < iterations; ++i) {- if (random().nextInt(5) == 0) {- forDeltaUtil.skip(in);- continue;- }- long base = 0;- final long[] restored = new long[ForUtil.BLOCK_SIZE];- forDeltaUtil.decodeAndPrefixSum(in, base, restored);- final long[] expected = new long[ForUtil.BLOCK_SIZE];- for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {- expected[j] = values[i * ForUtil.BLOCK_SIZE + j];- if (j > 0) {- expected[j] += expected[j - 1];- } else {- expected[j] += base;- }- }- assertArrayEquals(Arrays.toString(restored), expected, restored);- }- assertEquals(endPointer, in.getFilePointer());- in.close();- }-- d.close();- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestForUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestForUtil.javadeleted file mode 100644index 75aa44c9eb3..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestForUtil.java+++ /dev/null@@ -1,97 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene84;--import com.carrotsearch.randomizedtesting.generators.RandomNumbers;-import java.io.IOException;-import java.util.Arrays;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.store.ByteBuffersDirectory;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.store.IndexOutput;-import org.apache.lucene.tests.util.LuceneTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;-import org.apache.lucene.tests.util.TestUtil;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.packed.PackedInts;--@Nightly // N-2 formats are only tested on nightly runs-public class TestForUtil extends LuceneTestCase {-- public void testEncodeDecode() throws IOException {- final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);- final int[] values = new int[iterations * ForUtil.BLOCK_SIZE];-- for (int i = 0; i < iterations; ++i) {- final int bpv = TestUtil.nextInt(random(), 1, 31);- for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {- values[i * ForUtil.BLOCK_SIZE + j] =- RandomNumbers.randomIntBetween(random(), 0, (int) PackedInts.maxValue(bpv));- }- }-- final Directory d = new ByteBuffersDirectory();- final long endPointer;-- {- // encode- IndexOutput out = EndiannessReverserUtil.createOutput(d, "test.bin", IOContext.DEFAULT);- final ForUtil forUtil = new ForUtil();-- for (int i = 0; i < iterations; ++i) {- long[] source = new long[ForUtil.BLOCK_SIZE];- long or = 0;- for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {- source[j] = values[i * ForUtil.BLOCK_SIZE + j];- or |= source[j];- }- final int bpv = PackedInts.bitsRequired(or);- out.writeByte((byte) bpv);- forUtil.encode(source, bpv, out);- }- endPointer = out.getFilePointer();- out.close();- }-- {- // decode- IndexInput in = EndiannessReverserUtil.openInput(d, "test.bin", IOContext.READONCE);- final ForUtil forUtil = new ForUtil();- for (int i = 0; i < iterations; ++i) {- final int bitsPerValue = in.readByte();- final long currentFilePointer = in.getFilePointer();- final long[] restored = new long[ForUtil.BLOCK_SIZE];- forUtil.decode(bitsPerValue, in, restored);- int[] ints = new int[ForUtil.BLOCK_SIZE];- for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {- ints[j] = Math.toIntExact(restored[j]);- }- assertArrayEquals(- Arrays.toString(ints),- ArrayUtil.copyOfSubArray(values, i * ForUtil.BLOCK_SIZE, (i + 1) * ForUtil.BLOCK_SIZE),- ints);- assertEquals(forUtil.numBytes(bitsPerValue), in.getFilePointer() - currentFilePointer);- }- assertEquals(endPointer, in.getFilePointer());- in.close();- }-- d.close();- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestLucene84PostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestLucene84PostingsFormat.javadeleted file mode 100644index 71b24c65b71..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestLucene84PostingsFormat.java+++ /dev/null@@ -1,137 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene84;--import java.io.IOException;-import java.util.ArrayList;-import java.util.Arrays;-import java.util.Collections;-import java.util.List;-import org.apache.lucene.backward_codecs.lucene40.blocktree.FieldReader;-import org.apache.lucene.backward_codecs.lucene40.blocktree.Stats;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.Codec;-import org.apache.lucene.codecs.CompetitiveImpactAccumulator;-import org.apache.lucene.codecs.Impact;-import org.apache.lucene.document.Document;-import org.apache.lucene.document.Field;-import org.apache.lucene.index.DirectoryReader;-import org.apache.lucene.index.FreqAndNormBuffer;-import org.apache.lucene.index.IndexWriter;-import org.apache.lucene.index.IndexWriterConfig;-import org.apache.lucene.store.ByteArrayDataInput;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.store.IndexOutput;-import org.apache.lucene.tests.analysis.MockAnalyzer;-import org.apache.lucene.tests.index.BasePostingsFormatTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;-import org.apache.lucene.tests.util.TestUtil;--@Nightly // N-2 formats are only tested on nightly runs-public class TestLucene84PostingsFormat extends BasePostingsFormatTestCase {- private final Codec codec = TestUtil.alwaysPostingsFormat(new Lucene84RWPostingsFormat());-- @Override- protected Codec getCodec() {- return codec;- }-- /** Make sure the final sub-block(s) are not skipped. */- public void testFinalBlock() throws Exception {- Directory d = newDirectory();- IndexWriter w = new IndexWriter(d, new IndexWriterConfig(new MockAnalyzer(random())));- for (int i = 0; i < 25; i++) {- Document doc = new Document();- doc.add(newStringField("field", Character.toString((char) (97 + i)), Field.Store.NO));- doc.add(newStringField("field", "z" + Character.toString((char) (97 + i)), Field.Store.NO));- w.addDocument(doc);- }- w.forceMerge(1);-- DirectoryReader r = DirectoryReader.open(w);- assertEquals(1, r.leaves().size());- FieldReader field = (FieldReader) r.leaves().get(0).reader().terms("field");- // We should see exactly two blocks: one root block (prefix empty string) and one block for z*- // terms (prefix z):- Stats stats = field.getStats();- assertEquals(0, stats.floorBlockCount);- assertEquals(2, stats.nonFloorBlockCount);- r.close();- w.close();- d.close();- }-- public void testImpactSerialization() throws IOException {- // omit norms and omit freqs- doTestImpactSerialization(Collections.singletonList(new Impact(1, 1L)));-- // omit freqs- doTestImpactSerialization(Collections.singletonList(new Impact(1, 42L)));- // omit freqs with very large norms- doTestImpactSerialization(Collections.singletonList(new Impact(1, -100L)));-- // omit norms- doTestImpactSerialization(Collections.singletonList(new Impact(30, 1L)));- // omit norms with large freq- doTestImpactSerialization(Collections.singletonList(new Impact(500, 1L)));-- // freqs and norms, basic- doTestImpactSerialization(- Arrays.asList(- new Impact(1, 7L),- new Impact(3, 9L),- new Impact(7, 10L),- new Impact(15, 11L),- new Impact(20, 13L),- new Impact(28, 14L)));-- // freqs and norms, high values- doTestImpactSerialization(- Arrays.asList(- new Impact(2, 2L),- new Impact(10, 10L),- new Impact(12, 50L),- new Impact(50, -100L),- new Impact(1000, -80L),- new Impact(1005, -3L)));- }-- private void doTestImpactSerialization(List<Impact> impacts) throws IOException {- CompetitiveImpactAccumulator acc = new CompetitiveImpactAccumulator();- for (Impact impact : impacts) {- acc.add(impact.freq, impact.norm);- }- try (Directory dir = newDirectory()) {- try (IndexOutput out = EndiannessReverserUtil.createOutput(dir, "foo", IOContext.DEFAULT)) {- Lucene84SkipWriter.writeImpacts(acc, out);- }- try (IndexInput in = EndiannessReverserUtil.openInput(dir, "foo", IOContext.DEFAULT)) {- byte[] b = new byte[Math.toIntExact(in.length())];- in.readBytes(b, 0, b.length);- FreqAndNormBuffer impacts2 =- Lucene84ScoreSkipReader.readImpacts(new ByteArrayDataInput(b), new FreqAndNormBuffer());- List<Impact> impacts2AsList = new ArrayList<>();- for (int i = 0; i < impacts2.size; ++i) {- impacts2AsList.add(new Impact(impacts2.freqs[i], impacts2.norms[i]));- }- assertEquals(impacts, impacts2AsList);- }- }- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestPForUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestPForUtil.javadeleted file mode 100644index fb6161830a3..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestPForUtil.java+++ /dev/null@@ -1,103 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene84;--import com.carrotsearch.randomizedtesting.generators.RandomNumbers;-import java.io.IOException;-import java.util.Arrays;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.store.ByteBuffersDirectory;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.store.IndexOutput;-import org.apache.lucene.tests.util.LuceneTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;-import org.apache.lucene.tests.util.TestUtil;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.packed.PackedInts;--@Nightly // N-2 formats are only tested on nightly runs-public class TestPForUtil extends LuceneTestCase {-- public void testEncodeDecode() throws IOException {- final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);- final int[] values = new int[iterations * ForUtil.BLOCK_SIZE];-- for (int i = 0; i < iterations; ++i) {- final int bpv = TestUtil.nextInt(random(), 0, 31);- for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {- values[i * ForUtil.BLOCK_SIZE + j] =- RandomNumbers.randomIntBetween(random(), 0, (int) PackedInts.maxValue(bpv));- if (random().nextInt(100) == 0) {- final int exceptionBpv;- if (random().nextInt(10) == 0) {- exceptionBpv = Math.min(bpv + TestUtil.nextInt(random(), 9, 16), 31);- } else {- exceptionBpv = Math.min(bpv + TestUtil.nextInt(random(), 1, 8), 31);- }- values[i * ForUtil.BLOCK_SIZE + j] |= random().nextInt(1 << (exceptionBpv - bpv)) << bpv;- }- }- }-- final Directory d = new ByteBuffersDirectory();- final long endPointer;-- {- // encode- IndexOutput out = EndiannessReverserUtil.createOutput(d, "test.bin", IOContext.DEFAULT);- final PForUtil pforUtil = new PForUtil(new ForUtil());-- for (int i = 0; i < iterations; ++i) {- long[] source = new long[ForUtil.BLOCK_SIZE];- for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {- source[j] = values[i * ForUtil.BLOCK_SIZE + j];- }- pforUtil.encode(source, out);- }- endPointer = out.getFilePointer();- out.close();- }-- {- // decode- IndexInput in = EndiannessReverserUtil.openInput(d, "test.bin", IOContext.READONCE);- final PForUtil pforUtil = new PForUtil(new ForUtil());- for (int i = 0; i < iterations; ++i) {- if (random().nextInt(5) == 0) {- pforUtil.skip(in);- continue;- }- final long[] restored = new long[ForUtil.BLOCK_SIZE];- pforUtil.decode(in, restored);- int[] ints = new int[ForUtil.BLOCK_SIZE];- for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {- ints[j] = Math.toIntExact(restored[j]);- }- assertArrayEquals(- Arrays.toString(ints),- ArrayUtil.copyOfSubArray(values, i * ForUtil.BLOCK_SIZE, (i + 1) * ForUtil.BLOCK_SIZE),- ints);- }- assertEquals(endPointer, in.getFilePointer());- in.close();- }-- d.close();- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86PointsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86PointsWriter.javadeleted file mode 100644index d0bcc935811..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86PointsWriter.java+++ /dev/null@@ -1,304 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene86;--import java.io.IOException;-import java.util.ArrayList;-import java.util.List;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.codecs.MutablePointTree;-import org.apache.lucene.codecs.PointsReader;-import org.apache.lucene.codecs.PointsWriter;-import org.apache.lucene.index.FieldInfo;-import org.apache.lucene.index.FieldInfos;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.MergeState;-import org.apache.lucene.index.PointValues;-import org.apache.lucene.index.PointValues.IntersectVisitor;-import org.apache.lucene.index.PointValues.Relation;-import org.apache.lucene.index.SegmentWriteState;-import org.apache.lucene.store.IndexOutput;-import org.apache.lucene.util.IORunnable;-import org.apache.lucene.util.IOUtils;-import org.apache.lucene.util.bkd.BKDConfig;-import org.apache.lucene.util.bkd.BKDWriter;--/** Writes dimensional values */-public class Lucene86PointsWriter extends PointsWriter {-- /** Outputs used to write the BKD tree data files. */- protected final IndexOutput metaOut, indexOut, dataOut;-- final SegmentWriteState writeState;- final int maxPointsInLeafNode;- final double maxMBSortInHeap;- private boolean finished;-- /** Full constructor */- public Lucene86PointsWriter(- SegmentWriteState writeState, int maxPointsInLeafNode, double maxMBSortInHeap)- throws IOException {- assert writeState.fieldInfos.hasPointValues();- this.writeState = writeState;- this.maxPointsInLeafNode = maxPointsInLeafNode;- this.maxMBSortInHeap = maxMBSortInHeap;- String dataFileName =- IndexFileNames.segmentFileName(- writeState.segmentInfo.name,- writeState.segmentSuffix,- Lucene86PointsFormat.DATA_EXTENSION);- dataOut =- EndiannessReverserUtil.createOutput(writeState.directory, dataFileName, writeState.context);- boolean success = false;- try {- CodecUtil.writeIndexHeader(- dataOut,- Lucene86PointsFormat.DATA_CODEC_NAME,- Lucene86PointsFormat.VERSION_CURRENT,- writeState.segmentInfo.getId(),- writeState.segmentSuffix);-- String metaFileName =- IndexFileNames.segmentFileName(- writeState.segmentInfo.name,- writeState.segmentSuffix,- Lucene86PointsFormat.META_EXTENSION);- metaOut =- EndiannessReverserUtil.createOutput(- writeState.directory, metaFileName, writeState.context);- CodecUtil.writeIndexHeader(- metaOut,- Lucene86PointsFormat.META_CODEC_NAME,- Lucene86PointsFormat.VERSION_CURRENT,- writeState.segmentInfo.getId(),- writeState.segmentSuffix);-- String indexFileName =- IndexFileNames.segmentFileName(- writeState.segmentInfo.name,- writeState.segmentSuffix,- Lucene86PointsFormat.INDEX_EXTENSION);- indexOut =- EndiannessReverserUtil.createOutput(- writeState.directory, indexFileName, writeState.context);- CodecUtil.writeIndexHeader(- indexOut,- Lucene86PointsFormat.INDEX_CODEC_NAME,- Lucene86PointsFormat.VERSION_CURRENT,- writeState.segmentInfo.getId(),- writeState.segmentSuffix);-- success = true;- } finally {- if (success == false) {- IOUtils.closeWhileHandlingException(this);- }- }- }-- /**- * Uses the defaults values for {@code maxPointsInLeafNode} (512) and {@code maxMBSortInHeap}- * (16.0)- */- public Lucene86PointsWriter(SegmentWriteState writeState) throws IOException {- this(- writeState,- BKDConfig.DEFAULT_MAX_POINTS_IN_LEAF_NODE,- BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP);- }-- @Override- public void writeField(FieldInfo fieldInfo, PointsReader reader) throws IOException {-- PointValues.PointTree values = reader.getValues(fieldInfo.name).getPointTree();-- BKDConfig config =- new BKDConfig(- fieldInfo.getPointDimensionCount(),- fieldInfo.getPointIndexDimensionCount(),- fieldInfo.getPointNumBytes(),- maxPointsInLeafNode);-- try (BKDWriter writer =- new BKDWriter(- writeState.segmentInfo.maxDoc(),- writeState.directory,- writeState.segmentInfo.name,- config,- maxMBSortInHeap,- values.size())) {-- if (values instanceof MutablePointTree) {- IORunnable finalizer =- writer.writeField(- metaOut, indexOut, dataOut, fieldInfo.name, (MutablePointTree) values);- if (finalizer != null) {- metaOut.writeInt(fieldInfo.number);- finalizer.run();- }- return;- }-- values.visitDocValues(- new IntersectVisitor() {- @Override- public void visit(int docID) {- throw new IllegalStateException();- }-- @Override- public void visit(int docID, byte[] packedValue) throws IOException {- writer.add(packedValue, docID);- }-- @Override- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {- return Relation.CELL_CROSSES_QUERY;- }- });-- // We could have 0 points on merge since all docs with dimensional fields may be deleted:- IORunnable finalizer = writer.finish(metaOut, indexOut, dataOut);- if (finalizer != null) {- metaOut.writeInt(fieldInfo.number);- finalizer.run();- }- }- }-- @Override- public void merge(MergeState mergeState) throws IOException {- /*- * If indexSort is activated and some of the leaves are not sorted the next test will catch that- * and the non-optimized merge will run. If the readers are all sorted then it's safe to perform- * a bulk merge of the points.- */- for (PointsReader reader : mergeState.pointsReaders) {- if (reader instanceof Lucene86PointsReader == false) {- // We can only bulk merge when all to-be-merged segments use our format:- super.merge(mergeState);- return;- }- }- for (PointsReader reader : mergeState.pointsReaders) {- if (reader != null) {- reader.checkIntegrity();- }- }-- for (FieldInfo fieldInfo : mergeState.mergeFieldInfos) {- if (fieldInfo.getPointDimensionCount() != 0) {- if (fieldInfo.getPointDimensionCount() == 1) {-- // Worst case total maximum size (if none of the points are deleted):- long totMaxSize = 0;- for (int i = 0; i < mergeState.pointsReaders.length; i++) {- PointsReader reader = mergeState.pointsReaders[i];- if (reader != null) {- FieldInfos readerFieldInfos = mergeState.fieldInfos[i];- FieldInfo readerFieldInfo = readerFieldInfos.fieldInfo(fieldInfo.name);- if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {- PointValues values = reader.getValues(fieldInfo.name);- if (values != null) {- totMaxSize += values.size();- }- }- }- }-- BKDConfig config =- new BKDConfig(- fieldInfo.getPointDimensionCount(),- fieldInfo.getPointIndexDimensionCount(),- fieldInfo.getPointNumBytes(),- maxPointsInLeafNode);-- // System.out.println("MERGE: field=" + fieldInfo.name);- // Optimize the 1D case to use BKDWriter.merge, which does a single merge sort of the- // already sorted incoming segments, instead of trying to sort all points again as if- // we were simply reindexing them:- try (BKDWriter writer =- new BKDWriter(- writeState.segmentInfo.maxDoc(),- writeState.directory,- writeState.segmentInfo.name,- config,- maxMBSortInHeap,- totMaxSize)) {- List<PointValues> pointValues = new ArrayList<>();- List<MergeState.DocMap> docMaps = new ArrayList<>();- for (int i = 0; i < mergeState.pointsReaders.length; i++) {- PointsReader reader = mergeState.pointsReaders[i];-- if (reader != null) {-- // we confirmed this up above- assert reader instanceof Lucene86PointsReader;- Lucene86PointsReader reader86 = (Lucene86PointsReader) reader;-- // NOTE: we cannot just use the merged fieldInfo.number (instead of resolving to- // this- // reader's FieldInfo as we do below) because field numbers can easily be different- // when addIndexes(Directory...) copies over segments from another index:-- FieldInfos readerFieldInfos = mergeState.fieldInfos[i];- FieldInfo readerFieldInfo = readerFieldInfos.fieldInfo(fieldInfo.name);- if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {- PointValues aPointValues = reader86.getValues(readerFieldInfo.name);- if (aPointValues != null) {- pointValues.add(aPointValues);- docMaps.add(mergeState.docMaps[i]);- }- }- }- }-- IORunnable finalizer = writer.merge(metaOut, indexOut, dataOut, docMaps, pointValues);- if (finalizer != null) {- metaOut.writeInt(fieldInfo.number);- finalizer.run();- }- }- } else {- mergeOneField(mergeState, fieldInfo);- }- }- }-- finish();- }-- @Override- public void finish() throws IOException {- if (finished) {- throw new IllegalStateException("already finished");- }- finished = true;- metaOut.writeInt(-1);- CodecUtil.writeFooter(indexOut);- CodecUtil.writeFooter(dataOut);- metaOut.writeLong(indexOut.getFilePointer());- metaOut.writeLong(dataOut.getFilePointer());- CodecUtil.writeFooter(metaOut);- }-- @Override- public void close() throws IOException {- IOUtils.close(metaOut, indexOut, dataOut);- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86RWCodec.javadeleted file mode 100644index 60cf10b2dd8..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86RWCodec.java+++ /dev/null@@ -1,92 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene86;--import org.apache.lucene.backward_codecs.lucene50.Lucene50RWCompoundFormat;-import org.apache.lucene.backward_codecs.lucene50.Lucene50RWStoredFieldsFormat;-import org.apache.lucene.backward_codecs.lucene50.Lucene50RWTermVectorsFormat;-import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat;-import org.apache.lucene.backward_codecs.lucene80.Lucene80RWNormsFormat;-import org.apache.lucene.backward_codecs.lucene84.Lucene84RWPostingsFormat;-import org.apache.lucene.codecs.CompoundFormat;-import org.apache.lucene.codecs.NormsFormat;-import org.apache.lucene.codecs.PointsFormat;-import org.apache.lucene.codecs.PostingsFormat;-import org.apache.lucene.codecs.SegmentInfoFormat;-import org.apache.lucene.codecs.StoredFieldsFormat;-import org.apache.lucene.codecs.TermVectorsFormat;-import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;--/** RW impersonation of {@link Lucene86Codec}. */-public class Lucene86RWCodec extends Lucene86Codec {-- private final StoredFieldsFormat storedFieldsFormat;- private final PostingsFormat defaultPF = new Lucene84RWPostingsFormat();- private final CompoundFormat compoundFormat = new Lucene50RWCompoundFormat();- private final PostingsFormat postingsFormat =- new PerFieldPostingsFormat() {- @Override- public PostingsFormat getPostingsFormatForField(String field) {- return defaultPF;- }- };-- /** No arguments constructor. */- public Lucene86RWCodec() {- storedFieldsFormat = new Lucene50RWStoredFieldsFormat();- }-- /** Constructor that takes a mode. */- public Lucene86RWCodec(Lucene50StoredFieldsFormat.Mode mode) {- storedFieldsFormat = new Lucene50RWStoredFieldsFormat(mode);- }-- @Override- public SegmentInfoFormat segmentInfoFormat() {- return new Lucene86RWSegmentInfoFormat();- }-- @Override- public StoredFieldsFormat storedFieldsFormat() {- return storedFieldsFormat;- }-- @Override- public NormsFormat normsFormat() {- return new Lucene80RWNormsFormat();- }-- @Override- public PostingsFormat postingsFormat() {- return postingsFormat;- }-- @Override- public final CompoundFormat compoundFormat() {- return compoundFormat;- }-- @Override- public TermVectorsFormat termVectorsFormat() {- return new Lucene50RWTermVectorsFormat();- }-- @Override- public final PointsFormat pointsFormat() {- return new Lucene86RWPointsFormat();- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86RWPointsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86RWPointsFormat.javadeleted file mode 100644index 9fd239d090d..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86RWPointsFormat.java+++ /dev/null@@ -1,33 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene86;--import java.io.IOException;-import org.apache.lucene.codecs.PointsWriter;-import org.apache.lucene.index.SegmentWriteState;--/** Writable version of Lucene86PointsFormat for testing */-public final class Lucene86RWPointsFormat extends Lucene86PointsFormat {-- /** Sole constructor */- public Lucene86RWPointsFormat() {}-- @Override- public PointsWriter fieldsWriter(SegmentWriteState state) throws IOException {- return new Lucene86PointsWriter(state);- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86RWSegmentInfoFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86RWSegmentInfoFormat.javadeleted file mode 100644index ca90cec4c6a..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86RWSegmentInfoFormat.java+++ /dev/null@@ -1,107 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--package org.apache.lucene.backward_codecs.lucene86;--import java.io.IOException;-import java.util.Set;-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.index.IndexFileNames;-import org.apache.lucene.index.IndexSorter;-import org.apache.lucene.index.SegmentInfo;-import org.apache.lucene.index.SortFieldProvider;-import org.apache.lucene.search.Sort;-import org.apache.lucene.search.SortField;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IndexOutput;-import org.apache.lucene.util.Version;--/** Writable version of Lucene86SegmentInfoFormat for testing */-public class Lucene86RWSegmentInfoFormat extends Lucene86SegmentInfoFormat {-- /** Sole constructor. */- public Lucene86RWSegmentInfoFormat() {}-- @Override- public void write(Directory dir, SegmentInfo si, IOContext ioContext) throws IOException {- final String fileName = IndexFileNames.segmentFileName(si.name, "", SI_EXTENSION);-- try (IndexOutput output = EndiannessReverserUtil.createOutput(dir, fileName, ioContext)) {- // Only add the file once we've successfully created it, else IFD assert can trip:- si.addFile(fileName);- CodecUtil.writeIndexHeader(output, CODEC_NAME, VERSION_CURRENT, si.getId(), "");-- writeSegmentInfo(output, si);-- CodecUtil.writeFooter(output);- }- }-- private void writeSegmentInfo(DataOutput output, SegmentInfo si) throws IOException {- Version version = si.getVersion();- if (version.major < 7) {- throw new IllegalArgumentException(- "invalid major version: should be >= 7 but got: " + version.major + " segment=" + si);- }- // Write the Lucene version that created this segment, since 3.1- output.writeInt(version.major);- output.writeInt(version.minor);- output.writeInt(version.bugfix);-- // Write the min Lucene version that contributed docs to the segment, since 7.0- if (si.getMinVersion() != null) {- output.writeByte((byte) 1);- Version minVersion = si.getMinVersion();- output.writeInt(minVersion.major);- output.writeInt(minVersion.minor);- output.writeInt(minVersion.bugfix);- } else {- output.writeByte((byte) 0);- }-- assert version.prerelease == 0;- output.writeInt(si.maxDoc());-- output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO));- output.writeMapOfStrings(si.getDiagnostics());- Set<String> files = si.files();- for (String file : files) {- if (!IndexFileNames.parseSegmentName(file).equals(si.name)) {- throw new IllegalArgumentException(- "invalid files: expected segment=" + si.name + ", got=" + files);- }- }- output.writeSetOfStrings(files);- output.writeMapOfStrings(si.getAttributes());-- Sort indexSort = si.getIndexSort();- int numSortFields = indexSort == null ? 0 : indexSort.getSort().length;- output.writeVInt(numSortFields);- for (int i = 0; i < numSortFields; ++i) {- SortField sortField = indexSort.getSort()[i];- IndexSorter sorter = sortField.getIndexSorter();- if (sorter == null) {- throw new IllegalArgumentException("cannot serialize SortField " + sortField);- }- output.writeString(sorter.getProviderName());- SortFieldProvider.write(sortField, output);- }- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/TestLucene86PointsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/TestLucene86PointsFormat.javadeleted file mode 100644index b59ec079d4a..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/TestLucene86PointsFormat.java+++ /dev/null@@ -1,361 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene86;--import java.io.IOException;-import java.util.Arrays;-import org.apache.lucene.backward_codecs.lucene87.Lucene87RWCodec;-import org.apache.lucene.codecs.Codec;-import org.apache.lucene.codecs.FilterCodec;-import org.apache.lucene.codecs.PointsFormat;-import org.apache.lucene.codecs.PointsReader;-import org.apache.lucene.codecs.PointsWriter;-import org.apache.lucene.document.BinaryPoint;-import org.apache.lucene.document.Document;-import org.apache.lucene.index.DirectoryReader;-import org.apache.lucene.index.IndexReader;-import org.apache.lucene.index.IndexWriter;-import org.apache.lucene.index.IndexWriterConfig;-import org.apache.lucene.index.LeafReader;-import org.apache.lucene.index.PointValues;-import org.apache.lucene.index.PointValues.IntersectVisitor;-import org.apache.lucene.index.PointValues.Relation;-import org.apache.lucene.index.SegmentReadState;-import org.apache.lucene.index.SegmentWriteState;-import org.apache.lucene.store.Directory;-import org.apache.lucene.tests.index.BasePointsFormatTestCase;-import org.apache.lucene.tests.index.MockRandomMergePolicy;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;-import org.apache.lucene.tests.util.TestUtil;-import org.apache.lucene.util.bkd.BKDConfig;--@Nightly // N-2 formats are only tested on nightly runs-public class TestLucene86PointsFormat extends BasePointsFormatTestCase {-- private final Codec codec;- private final int maxPointsInLeafNode;-- public TestLucene86PointsFormat() {- // standard issue- Codec defaultCodec = new Lucene87RWCodec();- if (random().nextBoolean()) {- // randomize parameters- maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 500);- double maxMBSortInHeap = 3.0 + (3 * random().nextDouble());- if (VERBOSE) {- System.out.println(- "TEST: using Lucene60PointsFormat with maxPointsInLeafNode="- + maxPointsInLeafNode- + " and maxMBSortInHeap="- + maxMBSortInHeap);- }-- // sneaky impersonation!- codec =- new FilterCodec(defaultCodec.getName(), defaultCodec) {- @Override- public PointsFormat pointsFormat() {- return new PointsFormat() {- @Override- public PointsWriter fieldsWriter(SegmentWriteState writeState) throws IOException {- return new Lucene86PointsWriter(writeState, maxPointsInLeafNode, maxMBSortInHeap);- }-- @Override- public PointsReader fieldsReader(SegmentReadState readState) throws IOException {- return new Lucene86PointsReader(readState);- }- };- }- };- } else {- // standard issue- codec = defaultCodec;- maxPointsInLeafNode = BKDConfig.DEFAULT_MAX_POINTS_IN_LEAF_NODE;- }- }-- @Override- protected Codec getCodec() {- return codec;- }-- @Override- public void testMergeStability() throws Exception {- assumeFalse(- "TODO: mess with the parameters and test gets angry!", codec instanceof FilterCodec);- super.testMergeStability();- }-- // TODO: clean up the math/estimation here rather than suppress so many warnings- @SuppressWarnings({"NarrowCalculation", "LongDoubleConversion"})- public void testEstimatePointCount() throws IOException {- Directory dir = newDirectory();- IndexWriterConfig iwc = newIndexWriterConfig();- // Avoid mockRandomMP since it may cause non-optimal merges that make the- // number of points per leaf hard to predict- while (iwc.getMergePolicy() instanceof MockRandomMergePolicy) {- iwc.setMergePolicy(newMergePolicy());- }- IndexWriter w = new IndexWriter(dir, iwc);- byte[] pointValue = new byte[3];- byte[] uniquePointValue = new byte[3];- random().nextBytes(uniquePointValue);- final int numDocs =- TEST_NIGHTLY ? atLeast(10000) : atLeast(500); // at night, make sure we have several leaves- final boolean multiValues = random().nextBoolean();- int totalValues = 0;- for (int i = 0; i < numDocs; ++i) {- Document doc = new Document();- if (i == numDocs / 2) {- totalValues++;- doc.add(new BinaryPoint("f", uniquePointValue));- } else {- final int numValues = (multiValues) ? TestUtil.nextInt(random(), 2, 100) : 1;- for (int j = 0; j < numValues; j++) {- do {- random().nextBytes(pointValue);- } while (Arrays.equals(pointValue, uniquePointValue));- doc.add(new BinaryPoint("f", pointValue));- totalValues++;- }- }- w.addDocument(doc);- }- w.forceMerge(1);- final IndexReader r = DirectoryReader.open(w);- w.close();- final LeafReader lr = getOnlyLeafReader(r);- PointValues points = lr.getPointValues("f");-- IntersectVisitor allPointsVisitor =- new IntersectVisitor() {- @Override- public void visit(int docID, byte[] packedValue) throws IOException {}-- @Override- public void visit(int docID) throws IOException {}-- @Override- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {- return Relation.CELL_INSIDE_QUERY;- }- };-- assertEquals(totalValues, points.estimatePointCount(allPointsVisitor));- assertEquals(numDocs, points.estimateDocCount(allPointsVisitor));-- IntersectVisitor noPointsVisitor =- new IntersectVisitor() {- @Override- public void visit(int docID, byte[] packedValue) throws IOException {}-- @Override- public void visit(int docID) throws IOException {}-- @Override- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {- return Relation.CELL_OUTSIDE_QUERY;- }- };-- // Return 0 if no points match- assertEquals(0, points.estimatePointCount(noPointsVisitor));- assertEquals(0, points.estimateDocCount(noPointsVisitor));-- IntersectVisitor onePointMatchVisitor =- new IntersectVisitor() {- @Override- public void visit(int docID, byte[] packedValue) throws IOException {}-- @Override- public void visit(int docID) throws IOException {}-- @Override- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {- if (Arrays.compareUnsigned(uniquePointValue, 0, 3, maxPackedValue, 0, 3) > 0- || Arrays.compareUnsigned(uniquePointValue, 0, 3, minPackedValue, 0, 3) < 0) {- return Relation.CELL_OUTSIDE_QUERY;- }- return Relation.CELL_CROSSES_QUERY;- }- };-- // If only one point matches, then the point count is (maxPointsInLeafNode + 1) / 2- // in general, or maybe 2x that if the point is a split value- final long pointCount = points.estimatePointCount(onePointMatchVisitor);- final long lastNodePointCount = totalValues % maxPointsInLeafNode;- assertTrue(- "" + pointCount,- pointCount == (maxPointsInLeafNode + 1) / 2 // common case- || pointCount == (lastNodePointCount + 1) / 2 // not fully populated leaf- || pointCount == 2 * ((maxPointsInLeafNode + 1) / 2) // if the point is a split value- || pointCount- == ((maxPointsInLeafNode + 1) / 2)- + ((lastNodePointCount + 1)- / 2)); // if the point is a split value and one leaf is not fully populated-- final long docCount = points.estimateDocCount(onePointMatchVisitor);-- if (multiValues) {- assertEquals(- docCount,- (long)- (docCount- * (1d- - Math.pow(- (numDocs - pointCount) / points.size(), points.size() / docCount))));- } else {- assertEquals(Math.min(pointCount, numDocs), docCount);- }- r.close();- dir.close();- }-- // The tree is always balanced in the N dims case, and leaves are- // not all full so things are a bit different- // TODO: clean up the math/estimation here rather than suppress so many warnings- @SuppressWarnings({"NarrowCalculation", "LongDoubleConversion"})- public void testEstimatePointCount2Dims() throws IOException {- Directory dir = newDirectory();- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());- byte[][] pointValue = new byte[2][];- pointValue[0] = new byte[3];- pointValue[1] = new byte[3];- byte[][] uniquePointValue = new byte[2][];- uniquePointValue[0] = new byte[3];- uniquePointValue[1] = new byte[3];- random().nextBytes(uniquePointValue[0]);- random().nextBytes(uniquePointValue[1]);- final int numDocs =- TEST_NIGHTLY- ? atLeast(10000)- : atLeast(1000); // in nightly, make sure we have several leaves- final boolean multiValues = random().nextBoolean();- int totalValues = 0;- for (int i = 0; i < numDocs; ++i) {- Document doc = new Document();- if (i == numDocs / 2) {- doc.add(new BinaryPoint("f", uniquePointValue));- totalValues++;- } else {- final int numValues = (multiValues) ? TestUtil.nextInt(random(), 2, 100) : 1;- for (int j = 0; j < numValues; j++) {- do {- random().nextBytes(pointValue[0]);- random().nextBytes(pointValue[1]);- } while (Arrays.equals(pointValue[0], uniquePointValue[0])- || Arrays.equals(pointValue[1], uniquePointValue[1]));- doc.add(new BinaryPoint("f", pointValue));- totalValues++;- }- }- w.addDocument(doc);- }- w.forceMerge(1);- final IndexReader r = DirectoryReader.open(w);- w.close();- final LeafReader lr = getOnlyLeafReader(r);- PointValues points = lr.getPointValues("f");-- IntersectVisitor allPointsVisitor =- new IntersectVisitor() {- @Override- public void visit(int docID, byte[] packedValue) throws IOException {}-- @Override- public void visit(int docID) throws IOException {}-- @Override- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {- return Relation.CELL_INSIDE_QUERY;- }- };-- assertEquals(totalValues, points.estimatePointCount(allPointsVisitor));- assertEquals(numDocs, points.estimateDocCount(allPointsVisitor));-- IntersectVisitor noPointsVisitor =- new IntersectVisitor() {- @Override- public void visit(int docID, byte[] packedValue) throws IOException {}-- @Override- public void visit(int docID) throws IOException {}-- @Override- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {- return Relation.CELL_OUTSIDE_QUERY;- }- };-- // Return 0 if no points match- assertEquals(0, points.estimatePointCount(noPointsVisitor));- assertEquals(0, points.estimateDocCount(noPointsVisitor));-- IntersectVisitor onePointMatchVisitor =- new IntersectVisitor() {- @Override- public void visit(int docID, byte[] packedValue) throws IOException {}-- @Override- public void visit(int docID) throws IOException {}-- @Override- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {- for (int dim = 0; dim < 2; ++dim) {- if (Arrays.compareUnsigned(- uniquePointValue[dim], 0, 3, maxPackedValue, dim * 3, dim * 3 + 3)- > 0- || Arrays.compareUnsigned(- uniquePointValue[dim], 0, 3, minPackedValue, dim * 3, dim * 3 + 3)- < 0) {- return Relation.CELL_OUTSIDE_QUERY;- }- }- return Relation.CELL_CROSSES_QUERY;- }- };-- final long pointCount = points.estimatePointCount(onePointMatchVisitor);- final long lastNodePointCount = totalValues % maxPointsInLeafNode;- assertTrue(- "" + pointCount,- pointCount == (maxPointsInLeafNode + 1) / 2 // common case- || pointCount == (lastNodePointCount + 1) / 2 // not fully populated leaf- || pointCount == 2 * ((maxPointsInLeafNode + 1) / 2) // if the point is a split value- || pointCount == ((maxPointsInLeafNode + 1) / 2) + ((lastNodePointCount + 1) / 2)- // in extreme cases, a point can be shared by 4 leaves- || pointCount == 4 * ((maxPointsInLeafNode + 1) / 2)- || pointCount == 3 * ((maxPointsInLeafNode + 1) / 2) + ((lastNodePointCount + 1) / 2));-- final long docCount = points.estimateDocCount(onePointMatchVisitor);- if (multiValues) {- assertEquals(- docCount,- (long)- (docCount- * (1d- - Math.pow(- (numDocs - pointCount) / points.size(), points.size() / docCount))));- } else {- assertEquals(Math.min(pointCount, numDocs), docCount);- }- r.close();- dir.close();- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/TestLucene86SegmentInfoFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/TestLucene86SegmentInfoFormat.javadeleted file mode 100644index 7388dcf8210..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/TestLucene86SegmentInfoFormat.java+++ /dev/null@@ -1,43 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--package org.apache.lucene.backward_codecs.lucene86;--import org.apache.lucene.backward_codecs.lucene87.Lucene87RWCodec;-import org.apache.lucene.codecs.Codec;-import org.apache.lucene.tests.index.BaseSegmentInfoFormatTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;-import org.apache.lucene.util.Version;--@Nightly // N-2 formats are only tested on nightly runs-public class TestLucene86SegmentInfoFormat extends BaseSegmentInfoFormatTestCase {-- @Override- protected Version[] getVersions() {- return new Version[] {Version.fromBits(8, 8, 1)};- }-- @Override- protected Codec getCodec() {- return new Lucene87RWCodec();- }-- @Override- protected boolean supportsHasBlocks() {- return false;- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/Lucene87RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/Lucene87RWCodec.javadeleted file mode 100644index f7be2b69822..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/Lucene87RWCodec.java+++ /dev/null@@ -1,91 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene87;--import org.apache.lucene.backward_codecs.lucene50.Lucene50RWCompoundFormat;-import org.apache.lucene.backward_codecs.lucene50.Lucene50RWTermVectorsFormat;-import org.apache.lucene.backward_codecs.lucene80.Lucene80RWNormsFormat;-import org.apache.lucene.backward_codecs.lucene84.Lucene84RWPostingsFormat;-import org.apache.lucene.backward_codecs.lucene86.Lucene86RWPointsFormat;-import org.apache.lucene.backward_codecs.lucene86.Lucene86RWSegmentInfoFormat;-import org.apache.lucene.codecs.CompoundFormat;-import org.apache.lucene.codecs.NormsFormat;-import org.apache.lucene.codecs.PointsFormat;-import org.apache.lucene.codecs.PostingsFormat;-import org.apache.lucene.codecs.SegmentInfoFormat;-import org.apache.lucene.codecs.StoredFieldsFormat;-import org.apache.lucene.codecs.TermVectorsFormat;-import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;--/** RW impersonation of {@link Lucene87Codec}. */-public class Lucene87RWCodec extends Lucene87Codec {-- private final PostingsFormat defaultPF = new Lucene84RWPostingsFormat();- private final CompoundFormat compoundFormat = new Lucene50RWCompoundFormat();- private final PostingsFormat postingsFormat =- new PerFieldPostingsFormat() {- @Override- public PostingsFormat getPostingsFormatForField(String field) {- return defaultPF;- }- };- private final Mode mode;-- public Lucene87RWCodec() {- this(Mode.BEST_SPEED);- }-- public Lucene87RWCodec(Mode mode) {- super(mode);- this.mode = mode;- }-- @Override- public SegmentInfoFormat segmentInfoFormat() {- return new Lucene86RWSegmentInfoFormat();- }-- @Override- public final CompoundFormat compoundFormat() {- return compoundFormat;- }-- @Override- public NormsFormat normsFormat() {- return new Lucene80RWNormsFormat();- }-- @Override- public PostingsFormat postingsFormat() {- return postingsFormat;- }-- @Override- public TermVectorsFormat termVectorsFormat() {- return new Lucene50RWTermVectorsFormat();- }-- @Override- public StoredFieldsFormat storedFieldsFormat() {- return new Lucene87RWStoredFieldsFormat(mode.storedMode);- }-- @Override- public final PointsFormat pointsFormat() {- return new Lucene86RWPointsFormat();- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/Lucene87RWStoredFieldsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/Lucene87RWStoredFieldsFormat.javadeleted file mode 100644index 2f2b347ff44..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/Lucene87RWStoredFieldsFormat.java+++ /dev/null@@ -1,69 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene87;--import java.io.IOException;-import org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50RWCompressingStoredFieldsFormat;-import org.apache.lucene.codecs.StoredFieldsFormat;-import org.apache.lucene.codecs.StoredFieldsWriter;-import org.apache.lucene.index.SegmentInfo;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;--/** RW impersonation of Lucene87StoredFieldsFormat. */-public class Lucene87RWStoredFieldsFormat extends Lucene87StoredFieldsFormat {-- public Lucene87RWStoredFieldsFormat(Mode mode) {- super(mode);- }-- @Override- public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context)- throws IOException {- String previous = si.putAttribute(MODE_KEY, mode.name());- if (previous != null && previous.equals(mode.name()) == false) {- throw new IllegalStateException(- "found existing value for "- + MODE_KEY- + " for segment: "- + si.name- + "old="- + previous- + ", new="- + mode.name());- }- return impl(mode).fieldsWriter(directory, si, context);- }-- @Override- StoredFieldsFormat impl(Mode mode) {- switch (mode) {- case BEST_SPEED:- return new Lucene50RWCompressingStoredFieldsFormat(- "Lucene87StoredFieldsFastData", BEST_SPEED_MODE, BEST_SPEED_BLOCK_LENGTH, 1024, 10);- case BEST_COMPRESSION:- return new Lucene50RWCompressingStoredFieldsFormat(- "Lucene87StoredFieldsHighData",- BEST_COMPRESSION_MODE,- BEST_COMPRESSION_BLOCK_LENGTH,- 4096,- 10);- default:- throw new AssertionError();- }- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormat.javadeleted file mode 100644index c58727903f9..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormat.java+++ /dev/null@@ -1,29 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene87;--import org.apache.lucene.codecs.Codec;-import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;--@Nightly // N-2 formats are only tested on nightly runs-public class TestLucene87StoredFieldsFormat extends BaseStoredFieldsFormatTestCase {- @Override- protected Codec getCodec() {- return new Lucene87RWCodec();- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormatHighCompression.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormatHighCompression.javadeleted file mode 100644index 43881f5a395..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormatHighCompression.java+++ /dev/null@@ -1,85 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene87;--import com.carrotsearch.randomizedtesting.generators.RandomPicks;-import org.apache.lucene.codecs.Codec;-import org.apache.lucene.document.Document;-import org.apache.lucene.document.StoredField;-import org.apache.lucene.index.DirectoryReader;-import org.apache.lucene.index.IndexWriter;-import org.apache.lucene.index.IndexWriterConfig;-import org.apache.lucene.index.StoredFields;-import org.apache.lucene.store.Directory;-import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase;-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;--@Nightly // N-2 formats are only tested on nightly runs-public class TestLucene87StoredFieldsFormatHighCompression extends BaseStoredFieldsFormatTestCase {- @Override- protected Codec getCodec() {- return new Lucene87RWCodec(Lucene87Codec.Mode.BEST_COMPRESSION);- }-- /**- * Change compression params (leaving it the same for old segments) and tests that nothing breaks.- */- public void testMixedCompressions() throws Exception {- Directory dir = newDirectory();- for (int i = 0; i < 10; i++) {- IndexWriterConfig iwc = newIndexWriterConfig();- iwc.setCodec(- new Lucene87RWCodec(RandomPicks.randomFrom(random(), Lucene87Codec.Mode.values())));- IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig());- Document doc = new Document();- doc.add(new StoredField("field1", "value1"));- doc.add(new StoredField("field2", "value2"));- iw.addDocument(doc);- if (random().nextInt(4) == 0) {- iw.forceMerge(1);- }- iw.commit();- iw.close();- }-- DirectoryReader ir = DirectoryReader.open(dir);- assertEquals(10, ir.numDocs());- StoredFields storedFields = ir.storedFields();- for (int i = 0; i < 10; i++) {- Document doc = storedFields.document(i);- assertEquals("value1", doc.get("field1"));- assertEquals("value2", doc.get("field2"));- }- ir.close();- // checkindex- dir.close();- }-- public void testInvalidOptions() {- expectThrows(- NullPointerException.class,- () -> {- new Lucene87RWCodec(null);- });-- expectThrows(- NullPointerException.class,- () -> {- new Lucene87StoredFieldsFormat(null);- });- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormatMergeInstance.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormatMergeInstance.javadeleted file mode 100644index 7d7444cbed6..00000000000--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormatMergeInstance.java+++ /dev/null@@ -1,29 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene87;--import org.apache.lucene.tests.util.LuceneTestCase.Nightly;--/** Test the merge instance of the Lucene87 stored fields format. */-@Nightly // N-2 formats are only tested on nightly runs-public class TestLucene87StoredFieldsFormatMergeInstance extends TestLucene87StoredFieldsFormat {-- @Override- protected boolean shouldTestMergeInstance() {- return true;- }-}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestAncientIndicesCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestAncientIndicesCompatibility.javaindex 246e6fe01b2..2fff52dd1b7 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestAncientIndicesCompatibility.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestAncientIndicesCompatibility.java@@ -199,7 +199,7 @@ public class TestAncientIndicesCompatibility extends LuceneTestCase {checker.setInfoStream(new PrintStream(bos, false, UTF_8));checker.setLevel(CheckIndex.Level.MIN_LEVEL_FOR_INTEGRITY_CHECKS);CheckIndex.Status indexStatus = checker.checkIndex();- if (version.startsWith("8.") || version.startsWith("9.")) {+ if (version.startsWith("9.")) {assertTrue(indexStatus.clean);} else {assertFalse(indexStatus.clean);diff --git a/lucene/core/build.gradle b/lucene/core/build.gradleindex ae4f31261ae..68e240b6561 100644--- a/lucene/core/build.gradle+++ b/lucene/core/build.gradle@@ -59,9 +59,6 @@ createForUtilRegenerateTask(project, "ForUtil", "ForUtil.java", "gen_ForUtil.py"file("src/java/org/apache/lucene/codecs/lucene104"))configure(project(":lucene:backward-codecs")) {- createForUtilRegenerateTask(project, "ForUtil84", "ForUtil.java", "gen_ForUtil.py",- file("src/java/org/apache/lucene/backward_codecs/lucene84"))-createForUtilRegenerateTask(project, "ForUtil90", "ForUtil.java", "gen_ForUtil.py",file("src/java/org/apache/lucene/backward_codecs/lucene90"))