Skip to content

Apache Lucene open-source search software

Tests: Remove backward compatibility tests older than previous major version (#15712)

Alexander Reelsenaf85bbcf1c07
diff --git a/lucene/backward-codecs/src/generated/checksums/regenerateForUtil84.json b/lucene/backward-codecs/src/generated/checksums/regenerateForUtil84.json
deleted file mode 100644
index 7e3eacb2c29..00000000000
--- a/lucene/backward-codecs/src/generated/checksums/regenerateForUtil84.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/ForUtil.java": "c3bff5677f7d98fbb362018a4c1dbad4d670610f",
- "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/gen_ForUtil.py": "008d3e60b303fbb7c0e8c21f18748a6f384fbbc6"
-}
diff --git a/lucene/backward-codecs/src/java/module-info.java b/lucene/backward-codecs/src/java/module-info.java
index dac0bb2f59f..60a0b2897eb 100644
--- a/lucene/backward-codecs/src/java/module-info.java
+++ b/lucene/backward-codecs/src/java/module-info.java
@@ -21,15 +21,7 @@ module org.apache.lucene.backward_codecs {
exports org.apache.lucene.backward_codecs;
exports org.apache.lucene.backward_codecs.compressing;
- exports org.apache.lucene.backward_codecs.lucene40.blocktree;
- exports org.apache.lucene.backward_codecs.lucene50;
- exports org.apache.lucene.backward_codecs.lucene50.compressing;
- exports org.apache.lucene.backward_codecs.lucene60;
- exports org.apache.lucene.backward_codecs.lucene70;
exports org.apache.lucene.backward_codecs.lucene80;
- exports org.apache.lucene.backward_codecs.lucene84;
- exports org.apache.lucene.backward_codecs.lucene86;
- exports org.apache.lucene.backward_codecs.lucene87;
exports org.apache.lucene.backward_codecs.lucene90;
exports org.apache.lucene.backward_codecs.lucene90.blocktree;
exports org.apache.lucene.backward_codecs.lucene91;
@@ -48,8 +40,6 @@ module org.apache.lucene.backward_codecs {
provides org.apache.lucene.codecs.DocValuesFormat with
org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat;
provides org.apache.lucene.codecs.PostingsFormat with
- org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat,
- org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat,
org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat,
org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat,
org.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormat,
@@ -66,10 +56,6 @@ module org.apache.lucene.backward_codecs {
org.apache.lucene.backward_codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat,
org.apache.lucene.backward_codecs.lucene102.Lucene102HnswBinaryQuantizedVectorsFormat;
provides org.apache.lucene.codecs.Codec with
- org.apache.lucene.backward_codecs.lucene80.Lucene80Codec,
- org.apache.lucene.backward_codecs.lucene84.Lucene84Codec,
- org.apache.lucene.backward_codecs.lucene86.Lucene86Codec,
- org.apache.lucene.backward_codecs.lucene87.Lucene87Codec,
org.apache.lucene.backward_codecs.lucene90.Lucene90Codec,
org.apache.lucene.backward_codecs.lucene91.Lucene91Codec,
org.apache.lucene.backward_codecs.lucene92.Lucene92Codec,
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/CompressionAlgorithm.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/CompressionAlgorithm.java
deleted file mode 100644
index 469e773e233..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/CompressionAlgorithm.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene40.blocktree;
-
-import java.io.IOException;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.util.compress.LowercaseAsciiCompression;
-
-/** Compression algorithm used for suffixes of a block of terms. */
-enum CompressionAlgorithm {
- NO_COMPRESSION(0x00) {
-
- @Override
- void read(DataInput in, byte[] out, int len) throws IOException {
- in.readBytes(out, 0, len);
- }
- },
-
- LOWERCASE_ASCII(0x01) {
-
- @Override
- void read(DataInput in, byte[] out, int len) throws IOException {
- LowercaseAsciiCompression.decompress(in, out, len);
- }
- },
-
- LZ4(0x02) {
-
- @Override
- void read(DataInput in, byte[] out, int len) throws IOException {
- org.apache.lucene.util.compress.LZ4.decompress(
- EndiannessReverserUtil.wrapDataInput(in), len, out, 0);
- }
- };
-
- private static final CompressionAlgorithm[] BY_CODE = new CompressionAlgorithm[3];
-
- static {
- for (CompressionAlgorithm alg : CompressionAlgorithm.values()) {
- BY_CODE[alg.code] = alg;
- }
- }
-
- /** Look up a {@link CompressionAlgorithm} by its {@link CompressionAlgorithm#code}. */
- static final CompressionAlgorithm byCode(int code) {
- if (code < 0 || code >= BY_CODE.length) {
- throw new IllegalArgumentException("Illegal code for a compression algorithm: " + code);
- }
- return BY_CODE[code];
- }
-
- public final int code;
-
- private CompressionAlgorithm(int code) {
- this.code = code;
- }
-
- abstract void read(DataInput in, byte[] out, int len) throws IOException;
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/FieldReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/FieldReader.java
deleted file mode 100644
index b74f8b1ecdd..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/FieldReader.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene40.blocktree;
-
-import static org.apache.lucene.util.fst.FST.readMetadata;
-
-import java.io.IOException;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.automaton.CompiledAutomaton;
-import org.apache.lucene.util.fst.ByteSequenceOutputs;
-import org.apache.lucene.util.fst.FST;
-import org.apache.lucene.util.fst.OffHeapFSTStore;
-
-/**
- * BlockTree's implementation of {@link Terms}.
- *
- * @lucene.internal
- */
-public final class FieldReader extends Terms {
-
- // private final boolean DEBUG = BlockTreeTermsWriter.DEBUG;
-
- final long numTerms;
- final FieldInfo fieldInfo;
- final long sumTotalTermFreq;
- final long sumDocFreq;
- final int docCount;
- final long rootBlockFP;
- final BytesRef rootCode;
- final BytesRef minTerm;
- final BytesRef maxTerm;
- final Lucene40BlockTreeTermsReader parent;
-
- final FST<BytesRef> index;
-
- // private boolean DEBUG;
-
- FieldReader(
- Lucene40BlockTreeTermsReader parent,
- FieldInfo fieldInfo,
- long numTerms,
- BytesRef rootCode,
- long sumTotalTermFreq,
- long sumDocFreq,
- int docCount,
- long indexStartFP,
- IndexInput metaIn,
- IndexInput indexIn,
- BytesRef minTerm,
- BytesRef maxTerm)
- throws IOException {
- assert numTerms > 0;
- this.fieldInfo = fieldInfo;
- // DEBUG = BlockTreeTermsReader.DEBUG && fieldInfo.name.equals("id");
- this.parent = parent;
- this.numTerms = numTerms;
- this.sumTotalTermFreq = sumTotalTermFreq;
- this.sumDocFreq = sumDocFreq;
- this.docCount = docCount;
- this.rootCode = rootCode;
- this.minTerm = minTerm;
- this.maxTerm = maxTerm;
- // if (DEBUG) {
- // System.out.println("BTTR: seg=" + segment + " field=" + fieldInfo.name + " rootBlockCode="
- // + rootCode + " divisor=" + indexDivisor);
- // }
- rootBlockFP =
- (new ByteArrayDataInput(rootCode.bytes, rootCode.offset, rootCode.length)).readVLong()
- >>> Lucene40BlockTreeTermsReader.OUTPUT_FLAGS_NUM_BITS;
- // Initialize FST always off-heap.
- final FST.FSTMetadata<BytesRef> fstMetadata;
- if (metaIn == indexIn) { // Only true before Lucene 8.6
- final IndexInput clone = indexIn.clone();
- clone.seek(indexStartFP);
- fstMetadata = readMetadata(clone, ByteSequenceOutputs.getSingleton());
- // FST bytes actually only start after the metadata.
- indexStartFP = clone.getFilePointer();
- } else {
- fstMetadata = readMetadata(metaIn, ByteSequenceOutputs.getSingleton());
- }
- index = FST.fromFSTReader(fstMetadata, new OffHeapFSTStore(indexIn, indexStartFP, fstMetadata));
- /*
- * if (false) {
- * final String dotFileName = segment + "_" + fieldInfo.name + ".dot";
- * Writer w = new OutputStreamWriter(new FileOutputStream(dotFileName));
- * Util.toDot(index, w, false, false);
- * System.out.println("FST INDEX: SAVED to " + dotFileName);
- * w.close();
- * }
- */
- }
-
- @Override
- public BytesRef getMin() throws IOException {
- if (minTerm == null) {
- // Older index that didn't store min/maxTerm
- return super.getMin();
- } else {
- return minTerm;
- }
- }
-
- @Override
- public BytesRef getMax() throws IOException {
- if (maxTerm == null) {
- // Older index that didn't store min/maxTerm
- return super.getMax();
- } else {
- return maxTerm;
- }
- }
-
- /** For debugging -- used by CheckIndex too */
- @Override
- public Stats getStats() throws IOException {
- return new SegmentTermsEnum(this).computeBlockStats();
- }
-
- @Override
- public boolean hasFreqs() {
- return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
- }
-
- @Override
- public boolean hasOffsets() {
- return fieldInfo
- .getIndexOptions()
- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
- >= 0;
- }
-
- @Override
- public boolean hasPositions() {
- return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- }
-
- @Override
- public boolean hasPayloads() {
- return fieldInfo.hasPayloads();
- }
-
- @Override
- public TermsEnum iterator() throws IOException {
- return new SegmentTermsEnum(this);
- }
-
- @Override
- public long size() {
- return numTerms;
- }
-
- @Override
- public long getSumTotalTermFreq() {
- return sumTotalTermFreq;
- }
-
- @Override
- public long getSumDocFreq() {
- return sumDocFreq;
- }
-
- @Override
- public int getDocCount() {
- return docCount;
- }
-
- @Override
- public TermsEnum intersect(CompiledAutomaton compiled, BytesRef startTerm) throws IOException {
- // if (DEBUG) System.out.println(" FieldReader.intersect startTerm=" +
- // ToStringUtils.bytesRefToString(startTerm));
- // System.out.println("intersect: " + compiled.type + " a=" + compiled.automaton);
- // TODO: we could push "it's a range" or "it's a prefix" down into IntersectTermsEnum?
- // can we optimize knowing that...?
- if (compiled.type != CompiledAutomaton.AUTOMATON_TYPE.NORMAL) {
- throw new IllegalArgumentException("please use CompiledAutomaton.getTermsEnum instead");
- }
- return new IntersectTermsEnum(
- this,
- compiled.getTransitionAccessor(),
- compiled.getByteRunnable(),
- compiled.commonSuffixRef,
- startTerm);
- }
-
- @Override
- public String toString() {
- return "BlockTreeTerms(seg="
- + parent.segment
- + " terms="
- + numTerms
- + ",postings="
- + sumDocFreq
- + ",positions="
- + sumTotalTermFreq
- + ",docs="
- + docCount
- + ")";
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/IntersectTermsEnum.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/IntersectTermsEnum.java
deleted file mode 100644
index 80f47b1eeef..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/IntersectTermsEnum.java
+++ /dev/null
@@ -1,577 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene40.blocktree;
-
-import java.io.IOException;
-import org.apache.lucene.index.BaseTermsEnum;
-import org.apache.lucene.index.ImpactsEnum;
-import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.TermState;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.RamUsageEstimator;
-import org.apache.lucene.util.StringHelper;
-import org.apache.lucene.util.automaton.ByteRunnable;
-import org.apache.lucene.util.automaton.Transition;
-import org.apache.lucene.util.automaton.TransitionAccessor;
-import org.apache.lucene.util.fst.ByteSequenceOutputs;
-import org.apache.lucene.util.fst.FST;
-import org.apache.lucene.util.fst.Outputs;
-
-/**
- * This is used to implement efficient {@link Terms#intersect} for block-tree. Note that it cannot
- * seek, except for the initial term on init. It just "nexts" through the intersection of the
- * automaton and the terms. It does not use the terms index at all: on init, it loads the root
- * block, and scans its way to the initial term. Likewise, in next it scans until it finds a term
- * that matches the current automaton transition.
- */
-final class IntersectTermsEnum extends BaseTermsEnum {
-
- // static boolean DEBUG = BlockTreeTermsWriter.DEBUG;
-
- final IndexInput in;
- static final Outputs<BytesRef> fstOutputs = ByteSequenceOutputs.getSingleton();
-
- IntersectTermsEnumFrame[] stack;
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- private FST.Arc<BytesRef>[] arcs = new FST.Arc[5];
-
- final ByteRunnable runAutomaton;
- final TransitionAccessor automaton;
- final BytesRef commonSuffix;
-
- private IntersectTermsEnumFrame currentFrame;
- private Transition currentTransition;
-
- private final BytesRef term = new BytesRef();
-
- private final FST.BytesReader fstReader;
-
- final FieldReader fr;
-
- private BytesRef savedStartTerm;
-
- // TODO: in some cases we can filter by length? eg
- // regexp foo*bar must be at least length 6 bytes
- public IntersectTermsEnum(
- FieldReader fr,
- TransitionAccessor automaton,
- ByteRunnable runAutomaton,
- BytesRef commonSuffix,
- BytesRef startTerm)
- throws IOException {
- this.fr = fr;
-
- assert automaton != null;
- assert runAutomaton != null;
-
- this.runAutomaton = runAutomaton;
- this.automaton = automaton;
- this.commonSuffix = commonSuffix;
-
- in = fr.parent.termsIn.clone();
- stack = new IntersectTermsEnumFrame[5];
- for (int idx = 0; idx < stack.length; idx++) {
- stack[idx] = new IntersectTermsEnumFrame(this, idx);
- }
- for (int arcIdx = 0; arcIdx < arcs.length; arcIdx++) {
- arcs[arcIdx] = new FST.Arc<>();
- }
-
- fstReader = fr.index.getBytesReader();
-
- // TODO: if the automaton is "smallish" we really
- // should use the terms index to seek at least to
- // the initial term and likely to subsequent terms
- // (or, maybe just fallback to ATE for such cases).
- // Else the seek cost of loading the frames will be
- // too costly.
-
- final FST.Arc<BytesRef> arc = fr.index.getFirstArc(arcs[0]);
- // Empty string prefix must have an output in the index!
- assert arc.isFinal();
-
- // Special pushFrame since it's the first one:
- final IntersectTermsEnumFrame f = stack[0];
- f.fp = f.fpOrig = fr.rootBlockFP;
- f.prefix = 0;
- f.setState(0);
- f.arc = arc;
- f.outputPrefix = arc.output();
- f.load(fr.rootCode);
-
- // for assert:
- assert setSavedStartTerm(startTerm);
-
- currentFrame = f;
- if (startTerm != null) {
- seekToStartTerm(startTerm);
- }
- currentTransition = currentFrame.transition;
- }
-
- // only for assert:
- private boolean setSavedStartTerm(BytesRef startTerm) {
- savedStartTerm = startTerm == null ? null : BytesRef.deepCopyOf(startTerm);
- return true;
- }
-
- @Override
- public TermState termState() throws IOException {
- currentFrame.decodeMetaData();
- return currentFrame.termState.clone();
- }
-
- private IntersectTermsEnumFrame getFrame(int ord) throws IOException {
- if (ord >= stack.length) {
- final IntersectTermsEnumFrame[] next =
- new IntersectTermsEnumFrame
- [ArrayUtil.oversize(1 + ord, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
- System.arraycopy(stack, 0, next, 0, stack.length);
- for (int stackOrd = stack.length; stackOrd < next.length; stackOrd++) {
- next[stackOrd] = new IntersectTermsEnumFrame(this, stackOrd);
- }
- stack = next;
- }
- assert stack[ord].ord == ord;
- return stack[ord];
- }
-
- private FST.Arc<BytesRef> getArc(int ord) {
- if (ord >= arcs.length) {
- @SuppressWarnings({"rawtypes", "unchecked"})
- final FST.Arc<BytesRef>[] next =
- new FST.Arc[ArrayUtil.oversize(1 + ord, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
- System.arraycopy(arcs, 0, next, 0, arcs.length);
- for (int arcOrd = arcs.length; arcOrd < next.length; arcOrd++) {
- next[arcOrd] = new FST.Arc<>();
- }
- arcs = next;
- }
- return arcs[ord];
- }
-
- private IntersectTermsEnumFrame pushFrame(int state) throws IOException {
- assert currentFrame != null;
-
- final IntersectTermsEnumFrame f = getFrame(currentFrame == null ? 0 : 1 + currentFrame.ord);
-
- f.fp = f.fpOrig = currentFrame.lastSubFP;
- f.prefix = currentFrame.prefix + currentFrame.suffix;
- f.setState(state);
-
- // Walk the arc through the index -- we only
- // "bother" with this so we can get the floor data
- // from the index and skip floor blocks when
- // possible:
- FST.Arc<BytesRef> arc = currentFrame.arc;
- int idx = currentFrame.prefix;
- assert currentFrame.suffix > 0;
- BytesRef output = currentFrame.outputPrefix;
- while (idx < f.prefix) {
- final int target = term.bytes[idx] & 0xff;
- // TODO: we could be more efficient for the next()
- // case by using current arc as starting point,
- // passed to findTargetArc
- arc = fr.index.findTargetArc(target, arc, getArc(1 + idx), fstReader);
- assert arc != null;
- output = fstOutputs.add(output, arc.output());
- idx++;
- }
-
- f.arc = arc;
- f.outputPrefix = output;
- assert arc.isFinal();
- f.load(fstOutputs.add(output, arc.nextFinalOutput()));
- return f;
- }
-
- @Override
- public BytesRef term() {
- return term;
- }
-
- @Override
- public int docFreq() throws IOException {
- currentFrame.decodeMetaData();
- return currentFrame.termState.docFreq;
- }
-
- @Override
- public long totalTermFreq() throws IOException {
- currentFrame.decodeMetaData();
- return currentFrame.termState.totalTermFreq;
- }
-
- @Override
- public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException {
- currentFrame.decodeMetaData();
- return fr.parent.postingsReader.postings(fr.fieldInfo, currentFrame.termState, reuse, flags);
- }
-
- @Override
- public ImpactsEnum impacts(int flags) throws IOException {
- currentFrame.decodeMetaData();
- return fr.parent.postingsReader.impacts(fr.fieldInfo, currentFrame.termState, flags);
- }
-
- private int getState() {
- int state = currentFrame.state;
- for (int idx = 0; idx < currentFrame.suffix; idx++) {
- state =
- runAutomaton.step(
- state, currentFrame.suffixBytes[currentFrame.startBytePos + idx] & 0xff);
- assert state != -1;
- }
- return state;
- }
-
- // NOTE: specialized to only doing the first-time
- // seek, but we could generalize it to allow
- // arbitrary seekExact/Ceil. Note that this is a
- // seekFloor!
- private void seekToStartTerm(BytesRef target) throws IOException {
- assert currentFrame.ord == 0;
- if (term.length < target.length) {
- term.bytes = ArrayUtil.grow(term.bytes, target.length);
- }
- FST.Arc<BytesRef> arc = arcs[0];
- assert arc == currentFrame.arc;
-
- for (int idx = 0; idx <= target.length; idx++) {
-
- while (true) {
- final int savNextEnt = currentFrame.nextEnt;
- final int savePos = currentFrame.suffixesReader.getPosition();
- final int saveLengthPos = currentFrame.suffixLengthsReader.getPosition();
- final int saveStartBytePos = currentFrame.startBytePos;
- final int saveSuffix = currentFrame.suffix;
- final long saveLastSubFP = currentFrame.lastSubFP;
- final int saveTermBlockOrd = currentFrame.termState.termBlockOrd;
-
- final boolean isSubBlock = currentFrame.next();
-
- term.length = currentFrame.prefix + currentFrame.suffix;
- if (term.bytes.length < term.length) {
- term.bytes = ArrayUtil.grow(term.bytes, term.length);
- }
- System.arraycopy(
- currentFrame.suffixBytes,
- currentFrame.startBytePos,
- term.bytes,
- currentFrame.prefix,
- currentFrame.suffix);
-
- if (isSubBlock && StringHelper.startsWith(target, term)) {
- // Recurse
- currentFrame = pushFrame(getState());
- break;
- } else {
- final int cmp = term.compareTo(target);
- if (cmp < 0) {
- if (currentFrame.nextEnt == currentFrame.entCount) {
- if (!currentFrame.isLastInFloor) {
- // Advance to next floor block
- currentFrame.loadNextFloorBlock();
- continue;
- } else {
- return;
- }
- }
- continue;
- } else if (cmp == 0) {
- return;
- } else {
- // Fallback to prior entry: the semantics of
- // this method is that the first call to
- // next() will return the term after the
- // requested term
- currentFrame.nextEnt = savNextEnt;
- currentFrame.lastSubFP = saveLastSubFP;
- currentFrame.startBytePos = saveStartBytePos;
- currentFrame.suffix = saveSuffix;
- currentFrame.suffixesReader.setPosition(savePos);
- currentFrame.suffixLengthsReader.setPosition(saveLengthPos);
- currentFrame.termState.termBlockOrd = saveTermBlockOrd;
- System.arraycopy(
- currentFrame.suffixBytes,
- currentFrame.startBytePos,
- term.bytes,
- currentFrame.prefix,
- currentFrame.suffix);
- term.length = currentFrame.prefix + currentFrame.suffix;
- // If the last entry was a block we don't
- // need to bother recursing and pushing to
- // the last term under it because the first
- // next() will simply skip the frame anyway
- return;
- }
- }
- }
- }
-
- assert false;
- }
-
- private boolean popPushNext() throws IOException {
- // Pop finished frames
- while (currentFrame.nextEnt == currentFrame.entCount) {
- if (!currentFrame.isLastInFloor) {
- // Advance to next floor block
- currentFrame.loadNextFloorBlock();
- break;
- } else {
- if (currentFrame.ord == 0) {
- throw NoMoreTermsException.INSTANCE;
- }
- final long lastFP = currentFrame.fpOrig;
- currentFrame = stack[currentFrame.ord - 1];
- currentTransition = currentFrame.transition;
- assert currentFrame.lastSubFP == lastFP;
- }
- }
-
- return currentFrame.next();
- }
-
- // Only used internally when there are no more terms in next():
- private static final class NoMoreTermsException extends RuntimeException {
-
- // Only used internally when there are no more terms in next():
- public static final NoMoreTermsException INSTANCE = new NoMoreTermsException();
-
- private NoMoreTermsException() {}
-
- @Override
- public Throwable fillInStackTrace() {
- // Do nothing:
- return this;
- }
- }
-
- @Override
- public BytesRef next() throws IOException {
- try {
- return _next();
- } catch (NoMoreTermsException _) {
- // Provoke NPE if we are (illegally!) called again:
- currentFrame = null;
- return null;
- }
- }
-
- private BytesRef _next() throws IOException {
-
- boolean isSubBlock = popPushNext();
-
- nextTerm:
- while (true) {
- assert currentFrame.transition == currentTransition;
-
- int state;
- int lastState;
-
- // NOTE: suffix == 0 can only happen on the first term in a block, when
- // there is a term exactly matching a prefix in the index. If we
- // could somehow re-org the code so we only checked this case immediately
- // after pushing a frame...
- if (currentFrame.suffix != 0) {
-
- final byte[] suffixBytes = currentFrame.suffixBytes;
-
- // This is the first byte of the suffix of the term we are now on:
- final int label = suffixBytes[currentFrame.startBytePos] & 0xff;
-
- if (label < currentTransition.min) {
- // Common case: we are scanning terms in this block to "catch up" to
- // current transition in the automaton:
- int minTrans = currentTransition.min;
- while (currentFrame.nextEnt < currentFrame.entCount) {
- isSubBlock = currentFrame.next();
- if ((suffixBytes[currentFrame.startBytePos] & 0xff) >= minTrans) {
- continue nextTerm;
- }
- }
-
- // End of frame:
- isSubBlock = popPushNext();
- continue nextTerm;
- }
-
- // Advance where we are in the automaton to match this label:
-
- while (label > currentTransition.max) {
- if (currentFrame.transitionIndex >= currentFrame.transitionCount - 1) {
- // Pop this frame: no further matches are possible because
- // we've moved beyond what the max transition will allow
- if (currentFrame.ord == 0) {
- // Provoke NPE if we are (illegally!) called again:
- currentFrame = null;
- return null;
- }
- currentFrame = stack[currentFrame.ord - 1];
- currentTransition = currentFrame.transition;
- isSubBlock = popPushNext();
- continue nextTerm;
- }
- currentFrame.transitionIndex++;
- automaton.getNextTransition(currentTransition);
-
- if (label < currentTransition.min) {
- int minTrans = currentTransition.min;
- while (currentFrame.nextEnt < currentFrame.entCount) {
- isSubBlock = currentFrame.next();
- if ((suffixBytes[currentFrame.startBytePos] & 0xff) >= minTrans) {
- continue nextTerm;
- }
- }
-
- // End of frame:
- isSubBlock = popPushNext();
- continue nextTerm;
- }
- }
-
- if (commonSuffix != null && !isSubBlock) {
- final int termLen = currentFrame.prefix + currentFrame.suffix;
- if (termLen < commonSuffix.length) {
- // No match
- isSubBlock = popPushNext();
- continue nextTerm;
- }
-
- final byte[] commonSuffixBytes = commonSuffix.bytes;
-
- final int lenInPrefix = commonSuffix.length - currentFrame.suffix;
- assert commonSuffix.offset == 0;
- int suffixBytesPos;
- int commonSuffixBytesPos = 0;
-
- if (lenInPrefix > 0) {
- // A prefix of the common suffix overlaps with
- // the suffix of the block prefix so we first
- // test whether the prefix part matches:
- final byte[] termBytes = term.bytes;
- int termBytesPos = currentFrame.prefix - lenInPrefix;
- assert termBytesPos >= 0;
- final int termBytesPosEnd = currentFrame.prefix;
- while (termBytesPos < termBytesPosEnd) {
- if (termBytes[termBytesPos++] != commonSuffixBytes[commonSuffixBytesPos++]) {
- isSubBlock = popPushNext();
- continue nextTerm;
- }
- }
- suffixBytesPos = currentFrame.startBytePos;
- } else {
- suffixBytesPos = currentFrame.startBytePos + currentFrame.suffix - commonSuffix.length;
- }
-
- // Test overlapping suffix part:
- final int commonSuffixBytesPosEnd = commonSuffix.length;
- while (commonSuffixBytesPos < commonSuffixBytesPosEnd) {
- if (suffixBytes[suffixBytesPos++] != commonSuffixBytes[commonSuffixBytesPos++]) {
- isSubBlock = popPushNext();
- continue nextTerm;
- }
- }
- }
-
- // TODO: maybe we should do the same linear test
- // that AutomatonTermsEnum does, so that if we
- // reach a part of the automaton where .* is
- // "temporarily" accepted, we just blindly .next()
- // until the limit
-
- // See if the term suffix matches the automaton:
-
- // We know from above that the first byte in our suffix (label) matches
- // the current transition, so we step from the 2nd byte
- // in the suffix:
- lastState = currentFrame.state;
- state = currentTransition.dest;
-
- int end = currentFrame.startBytePos + currentFrame.suffix;
- for (int idx = currentFrame.startBytePos + 1; idx < end; idx++) {
- lastState = state;
- state = runAutomaton.step(state, suffixBytes[idx] & 0xff);
- if (state == -1) {
- // No match
- isSubBlock = popPushNext();
- continue nextTerm;
- }
- }
- } else {
- state = currentFrame.state;
- lastState = currentFrame.lastState;
- }
-
- if (isSubBlock) {
- // Match! Recurse:
- copyTerm();
- currentFrame = pushFrame(state);
- currentTransition = currentFrame.transition;
- currentFrame.lastState = lastState;
- } else if (runAutomaton.isAccept(state)) {
- copyTerm();
- assert savedStartTerm == null || term.compareTo(savedStartTerm) > 0
- : "saveStartTerm=" + savedStartTerm.utf8ToString() + " term=" + term.utf8ToString();
- return term;
- } else {
- // This term is a prefix of a term accepted by the automaton, but is not itself accepted
- }
-
- isSubBlock = popPushNext();
- }
- }
-
- private void copyTerm() {
- final int len = currentFrame.prefix + currentFrame.suffix;
- if (term.bytes.length < len) {
- term.bytes = ArrayUtil.grow(term.bytes, len);
- }
- System.arraycopy(
- currentFrame.suffixBytes,
- currentFrame.startBytePos,
- term.bytes,
- currentFrame.prefix,
- currentFrame.suffix);
- term.length = len;
- }
-
- @Override
- public boolean seekExact(BytesRef text) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void seekExact(long ord) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public long ord() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public SeekStatus seekCeil(BytesRef text) {
- throw new UnsupportedOperationException();
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/IntersectTermsEnumFrame.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/IntersectTermsEnumFrame.java
deleted file mode 100644
index d4c470c40a0..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/IntersectTermsEnumFrame.java
+++ /dev/null
@@ -1,357 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene40.blocktree;
-
-import java.io.IOException;
-import java.util.Arrays;
-import org.apache.lucene.codecs.BlockTermState;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.automaton.Transition;
-import org.apache.lucene.util.fst.FST;
-
-// TODO: can we share this with the frame in STE?
-final class IntersectTermsEnumFrame {
- final int ord;
- long fp;
- long fpOrig;
- long fpEnd;
- long lastSubFP;
-
- // private static boolean DEBUG = IntersectTermsEnum.DEBUG;
-
- // State in automaton
- int state;
-
- // State just before the last label
- int lastState;
-
- int metaDataUpto;
-
- byte[] suffixBytes = new byte[128];
- final ByteArrayDataInput suffixesReader = new ByteArrayDataInput();
-
- byte[] suffixLengthBytes;
- final ByteArrayDataInput suffixLengthsReader;
-
- byte[] statBytes = new byte[64];
- int statsSingletonRunLength = 0;
- final ByteArrayDataInput statsReader = new ByteArrayDataInput();
-
- byte[] floorData = new byte[32];
- final ByteArrayDataInput floorDataReader = new ByteArrayDataInput();
-
- // Length of prefix shared by all terms in this block
- int prefix;
-
- // Number of entries (term or sub-block) in this block
- int entCount;
-
- // Which term we will next read
- int nextEnt;
-
- // True if this block is either not a floor block,
- // or, it's the last sub-block of a floor block
- boolean isLastInFloor;
-
- // True if all entries are terms
- boolean isLeafBlock;
-
- int numFollowFloorBlocks;
- int nextFloorLabel;
-
- final Transition transition = new Transition();
- int transitionIndex;
- int transitionCount;
-
- FST.Arc<BytesRef> arc;
-
- final BlockTermState termState;
-
- // metadata buffer
- byte[] bytes = new byte[32];
-
- final ByteArrayDataInput bytesReader = new ByteArrayDataInput();
-
- // Cumulative output so far
- BytesRef outputPrefix;
-
- int startBytePos;
- int suffix;
-
- private final IntersectTermsEnum ite;
- private final int version;
-
- public IntersectTermsEnumFrame(IntersectTermsEnum ite, int ord) throws IOException {
- this.ite = ite;
- this.ord = ord;
- this.termState = ite.fr.parent.postingsReader.newTermState();
- this.termState.totalTermFreq = -1;
- this.version = ite.fr.parent.version;
- if (version >= Lucene40BlockTreeTermsReader.VERSION_COMPRESSED_SUFFIXES) {
- suffixLengthBytes = new byte[32];
- suffixLengthsReader = new ByteArrayDataInput();
- } else {
- suffixLengthBytes = null;
- suffixLengthsReader = suffixesReader;
- }
- }
-
- void loadNextFloorBlock() throws IOException {
- assert numFollowFloorBlocks > 0 : "nextFloorLabel=" + nextFloorLabel;
-
- do {
- fp = fpOrig + (floorDataReader.readVLong() >>> 1);
- numFollowFloorBlocks--;
- if (numFollowFloorBlocks != 0) {
- nextFloorLabel = floorDataReader.readByte() & 0xff;
- } else {
- nextFloorLabel = 256;
- }
- } while (numFollowFloorBlocks != 0 && nextFloorLabel <= transition.min);
-
- load(null);
- }
-
- public void setState(int state) {
- this.state = state;
- transitionIndex = 0;
- transitionCount = ite.automaton.getNumTransitions(state);
- if (transitionCount != 0) {
- ite.automaton.initTransition(state, transition);
- ite.automaton.getNextTransition(transition);
- } else {
-
- // Must set min to -1 so the "label < min" check never falsely triggers:
- transition.min = -1;
-
- // Must set max to -1 so we immediately realize we need to step to the next transition and
- // then pop this frame:
- transition.max = -1;
- }
- }
-
- void load(BytesRef frameIndexData) throws IOException {
- if (frameIndexData != null) {
- floorDataReader.reset(frameIndexData.bytes, frameIndexData.offset, frameIndexData.length);
- // Skip first long -- has redundant fp, hasTerms
- // flag, isFloor flag
- final long code = floorDataReader.readVLong();
- if ((code & Lucene40BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR) != 0) {
- // Floor frame
- numFollowFloorBlocks = floorDataReader.readVInt();
- nextFloorLabel = floorDataReader.readByte() & 0xff;
-
- // If current state is not accept, and has transitions, we must process
- // first block in case it has empty suffix:
- if (ite.runAutomaton.isAccept(state) == false && transitionCount != 0) {
- // Maybe skip floor blocks:
- assert transitionIndex == 0 : "transitionIndex=" + transitionIndex;
- while (numFollowFloorBlocks != 0 && nextFloorLabel <= transition.min) {
- fp = fpOrig + (floorDataReader.readVLong() >>> 1);
- numFollowFloorBlocks--;
- if (numFollowFloorBlocks != 0) {
- nextFloorLabel = floorDataReader.readByte() & 0xff;
- } else {
- nextFloorLabel = 256;
- }
- }
- }
- }
- }
-
- ite.in.seek(fp);
- int code = ite.in.readVInt();
- entCount = code >>> 1;
- assert entCount > 0;
- isLastInFloor = (code & 1) != 0;
-
- // term suffixes:
- if (version >= Lucene40BlockTreeTermsReader.VERSION_COMPRESSED_SUFFIXES) {
- final long codeL = ite.in.readVLong();
- isLeafBlock = (codeL & 0x04) != 0;
- final int numSuffixBytes = (int) (codeL >>> 3);
- if (suffixBytes.length < numSuffixBytes) {
- suffixBytes = new byte[ArrayUtil.oversize(numSuffixBytes, 1)];
- }
- final CompressionAlgorithm compressionAlg;
- try {
- compressionAlg = CompressionAlgorithm.byCode((int) codeL & 0x03);
- } catch (IllegalArgumentException e) {
- throw new CorruptIndexException(e.getMessage(), ite.in, e);
- }
- compressionAlg.read(ite.in, suffixBytes, numSuffixBytes);
- suffixesReader.reset(suffixBytes, 0, numSuffixBytes);
-
- int numSuffixLengthBytes = ite.in.readVInt();
- final boolean allEqual = (numSuffixLengthBytes & 0x01) != 0;
- numSuffixLengthBytes >>>= 1;
- if (suffixLengthBytes.length < numSuffixLengthBytes) {
- suffixLengthBytes = new byte[ArrayUtil.oversize(numSuffixLengthBytes, 1)];
- }
- if (allEqual) {
- Arrays.fill(suffixLengthBytes, 0, numSuffixLengthBytes, ite.in.readByte());
- } else {
- ite.in.readBytes(suffixLengthBytes, 0, numSuffixLengthBytes);
- }
- suffixLengthsReader.reset(suffixLengthBytes, 0, numSuffixLengthBytes);
- } else {
- code = ite.in.readVInt();
- isLeafBlock = (code & 1) != 0;
- int numBytes = code >>> 1;
- if (suffixBytes.length < numBytes) {
- suffixBytes = new byte[ArrayUtil.oversize(numBytes, 1)];
- }
- ite.in.readBytes(suffixBytes, 0, numBytes);
- suffixesReader.reset(suffixBytes, 0, numBytes);
- }
-
- // stats
- int numBytes = ite.in.readVInt();
- if (statBytes.length < numBytes) {
- statBytes = new byte[ArrayUtil.oversize(numBytes, 1)];
- }
- ite.in.readBytes(statBytes, 0, numBytes);
- statsReader.reset(statBytes, 0, numBytes);
- statsSingletonRunLength = 0;
- metaDataUpto = 0;
-
- termState.termBlockOrd = 0;
- nextEnt = 0;
-
- // metadata
- numBytes = ite.in.readVInt();
- if (bytes.length < numBytes) {
- bytes = new byte[ArrayUtil.oversize(numBytes, 1)];
- }
- ite.in.readBytes(bytes, 0, numBytes);
- bytesReader.reset(bytes, 0, numBytes);
-
- if (!isLastInFloor) {
- // Sub-blocks of a single floor block are always
- // written one after another -- tail recurse:
- fpEnd = ite.in.getFilePointer();
- }
- }
-
- // TODO: maybe add scanToLabel; should give perf boost
-
- // Decodes next entry; returns true if it's a sub-block
- public boolean next() {
- if (isLeafBlock) {
- nextLeaf();
- return false;
- } else {
- return nextNonLeaf();
- }
- }
-
- public void nextLeaf() {
- assert nextEnt != -1 && nextEnt < entCount
- : "nextEnt=" + nextEnt + " entCount=" + entCount + " fp=" + fp;
- nextEnt++;
- suffix = suffixLengthsReader.readVInt();
- startBytePos = suffixesReader.getPosition();
- suffixesReader.skipBytes(suffix);
- }
-
- public boolean nextNonLeaf() {
- assert nextEnt != -1 && nextEnt < entCount
- : "nextEnt=" + nextEnt + " entCount=" + entCount + " fp=" + fp;
- nextEnt++;
- final int code = suffixLengthsReader.readVInt();
- suffix = code >>> 1;
- startBytePos = suffixesReader.getPosition();
- suffixesReader.skipBytes(suffix);
- if ((code & 1) == 0) {
- // A normal term
- termState.termBlockOrd++;
- return false;
- } else {
- // A sub-block; make sub-FP absolute:
- lastSubFP = fp - suffixLengthsReader.readVLong();
- return true;
- }
- }
-
- public int getTermBlockOrd() {
- return isLeafBlock ? nextEnt : termState.termBlockOrd;
- }
-
- public void decodeMetaData() throws IOException {
-
- // lazily catch up on metadata decode:
- final int limit = getTermBlockOrd();
- boolean absolute = metaDataUpto == 0;
- assert limit > 0;
-
- // TODO: better API would be "jump straight to term=N"???
- while (metaDataUpto < limit) {
-
- // TODO: we could make "tiers" of metadata, ie,
- // decode docFreq/totalTF but don't decode postings
- // metadata; this way caller could get
- // docFreq/totalTF w/o paying decode cost for
- // postings
-
- // TODO: if docFreq were bulk decoded we could
- // just skipN here:
-
- // stats
- if (version >= Lucene40BlockTreeTermsReader.VERSION_COMPRESSED_SUFFIXES) {
- if (statsSingletonRunLength > 0) {
- termState.docFreq = 1;
- termState.totalTermFreq = 1;
- statsSingletonRunLength--;
- } else {
- int token = statsReader.readVInt();
- if (version >= Lucene40BlockTreeTermsReader.VERSION_COMPRESSED_SUFFIXES
- && (token & 1) == 1) {
- termState.docFreq = 1;
- termState.totalTermFreq = 1;
- statsSingletonRunLength = token >>> 1;
- } else {
- termState.docFreq = token >>> 1;
- if (ite.fr.fieldInfo.getIndexOptions() == IndexOptions.DOCS) {
- termState.totalTermFreq = termState.docFreq;
- } else {
- termState.totalTermFreq = termState.docFreq + statsReader.readVLong();
- }
- }
- }
- } else {
- termState.docFreq = statsReader.readVInt();
- // if (DEBUG) System.out.println(" dF=" + state.docFreq);
- if (ite.fr.fieldInfo.getIndexOptions() == IndexOptions.DOCS) {
- termState.totalTermFreq = termState.docFreq; // all postings have freq=1
- } else {
- termState.totalTermFreq = termState.docFreq + statsReader.readVLong();
- // if (DEBUG) System.out.println(" totTF=" + state.totalTermFreq);
- }
- }
- // metadata
- ite.fr.parent.postingsReader.decodeTerm(bytesReader, ite.fr.fieldInfo, termState, absolute);
-
- metaDataUpto++;
- absolute = false;
- }
- termState.termBlockOrd = metaDataUpto;
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/Lucene40BlockTreeTermsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/Lucene40BlockTreeTermsReader.java
deleted file mode 100644
index 8d578b8fdca..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/Lucene40BlockTreeTermsReader.java
+++ /dev/null
@@ -1,378 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene40.blocktree;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.FieldsProducer;
-import org.apache.lucene.codecs.PostingsReaderBase;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.fst.ByteSequenceOutputs;
-import org.apache.lucene.util.fst.Outputs;
-
-/**
- * A block-based terms index and dictionary that assigns terms to variable length blocks according
- * to how they share prefixes. The terms index is a prefix trie whose leaves are term blocks. The
- * advantage of this approach is that seekExact is often able to determine a term cannot exist
- * without doing any IO, and intersection with Automata is very fast. Note that this terms
- * dictionary has its own fixed terms index (ie, it does not support a pluggable terms index
- * implementation).
- *
- * <p><b>NOTE</b>: this terms dictionary supports min/maxItemsPerBlock during indexing to control
- * how much memory the terms index uses.
- *
- * <p>The data structure used by this implementation is very similar to a burst trie
- * (http://citeseer.ist.psu.edu/viewdoc/summary?doi=10.1.1.18.3499), but with added logic to break
- * up too-large blocks of all terms sharing a given prefix into smaller ones.
- *
- * <p>Use {@link org.apache.lucene.index.CheckIndex} with the <code>-verbose</code> option to see
- * summary statistics on the blocks in the dictionary.
- *
- * <p>See {@code BlockTreeTermsWriter}.
- *
- * @lucene.experimental
- */
-public final class Lucene40BlockTreeTermsReader extends FieldsProducer {
-
- static final Outputs<BytesRef> FST_OUTPUTS = ByteSequenceOutputs.getSingleton();
-
- static final BytesRef NO_OUTPUT = FST_OUTPUTS.getNoOutput();
-
- static final int OUTPUT_FLAGS_NUM_BITS = 2;
- static final int OUTPUT_FLAGS_MASK = 0x3;
- static final int OUTPUT_FLAG_IS_FLOOR = 0x1;
- static final int OUTPUT_FLAG_HAS_TERMS = 0x2;
-
- /** Extension of terms file */
- static final String TERMS_EXTENSION = "tim";
-
- static final String TERMS_CODEC_NAME = "BlockTreeTermsDict";
-
- /** Initial terms format. */
- public static final int VERSION_START = 3;
-
- /** The long[] + byte[] metadata has been replaced with a single byte[]. */
- public static final int VERSION_META_LONGS_REMOVED = 4;
-
- /** Suffixes are compressed to save space. */
- public static final int VERSION_COMPRESSED_SUFFIXES = 5;
-
- /** Metadata is written to its own file. */
- public static final int VERSION_META_FILE = 6;
-
- /** Current terms format. */
- public static final int VERSION_CURRENT = VERSION_META_FILE;
-
- /** Extension of terms index file */
- static final String TERMS_INDEX_EXTENSION = "tip";
-
- static final String TERMS_INDEX_CODEC_NAME = "BlockTreeTermsIndex";
-
- /** Extension of terms meta file */
- static final String TERMS_META_EXTENSION = "tmd";
-
- static final String TERMS_META_CODEC_NAME = "BlockTreeTermsMeta";
-
- // Open input to the main terms dict file (_X.tib)
- final IndexInput termsIn;
- // Open input to the terms index file (_X.tip)
- final IndexInput indexIn;
-
- // private static final boolean DEBUG = BlockTreeTermsWriter.DEBUG;
-
- // Reads the terms dict entries, to gather state to
- // produce DocsEnum on demand
- final PostingsReaderBase postingsReader;
-
- private final Map<String, FieldReader> fieldMap;
- private final List<String> fieldList;
-
- final String segment;
-
- final int version;
-
- /** Sole constructor. */
- public Lucene40BlockTreeTermsReader(PostingsReaderBase postingsReader, SegmentReadState state)
- throws IOException {
- boolean success = false;
-
- this.postingsReader = postingsReader;
- this.segment = state.segmentInfo.name;
-
- try {
- String termsName =
- IndexFileNames.segmentFileName(segment, state.segmentSuffix, TERMS_EXTENSION);
- termsIn = EndiannessReverserUtil.openInput(state.directory, termsName, state.context);
- version =
- CodecUtil.checkIndexHeader(
- termsIn,
- TERMS_CODEC_NAME,
- VERSION_START,
- VERSION_CURRENT,
- state.segmentInfo.getId(),
- state.segmentSuffix);
-
- String indexName =
- IndexFileNames.segmentFileName(segment, state.segmentSuffix, TERMS_INDEX_EXTENSION);
- indexIn = EndiannessReverserUtil.openInput(state.directory, indexName, state.context);
- CodecUtil.checkIndexHeader(
- indexIn,
- TERMS_INDEX_CODEC_NAME,
- version,
- version,
- state.segmentInfo.getId(),
- state.segmentSuffix);
-
- if (version < VERSION_META_FILE) {
- // Have PostingsReader init itself
- postingsReader.init(termsIn, state);
-
- // Verifying the checksum against all bytes would be too costly, but for now we at least
- // verify proper structure of the checksum footer. This is cheap and can detect some forms
- // of corruption such as file truncation.
- CodecUtil.retrieveChecksum(indexIn);
- CodecUtil.retrieveChecksum(termsIn);
- }
-
- // Read per-field details
- String metaName =
- IndexFileNames.segmentFileName(segment, state.segmentSuffix, TERMS_META_EXTENSION);
- Map<String, FieldReader> fieldMap = null;
- Throwable priorE = null;
- long indexLength = -1, termsLength = -1;
- try (ChecksumIndexInput metaIn =
- version >= VERSION_META_FILE
- ? EndiannessReverserUtil.openChecksumInput(state.directory, metaName, state.context)
- : null) {
- try {
- final IndexInput indexMetaIn, termsMetaIn;
- if (version >= VERSION_META_FILE) {
- CodecUtil.checkIndexHeader(
- metaIn,
- TERMS_META_CODEC_NAME,
- version,
- version,
- state.segmentInfo.getId(),
- state.segmentSuffix);
- indexMetaIn = termsMetaIn = metaIn;
- postingsReader.init(metaIn, state);
- } else {
- seekDir(termsIn);
- seekDir(termsIn);
- seekDir(indexIn);
- indexMetaIn = indexIn;
- termsMetaIn = termsIn;
- }
-
- final int numFields = termsMetaIn.readVInt();
- if (numFields < 0) {
- throw new CorruptIndexException("invalid numFields: " + numFields, termsMetaIn);
- }
- fieldMap = HashMap.newHashMap(numFields);
- for (int i = 0; i < numFields; ++i) {
- final int field = termsMetaIn.readVInt();
- final long numTerms = termsMetaIn.readVLong();
- if (numTerms <= 0) {
- throw new CorruptIndexException(
- "Illegal numTerms for field number: " + field, termsMetaIn);
- }
- final BytesRef rootCode = readBytesRef(termsMetaIn);
- final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
- if (fieldInfo == null) {
- throw new CorruptIndexException("invalid field number: " + field, termsMetaIn);
- }
- final long sumTotalTermFreq = termsMetaIn.readVLong();
- // when frequencies are omitted, sumDocFreq=sumTotalTermFreq and only one value is
- // written.
- final long sumDocFreq =
- fieldInfo.getIndexOptions() == IndexOptions.DOCS
- ? sumTotalTermFreq
- : termsMetaIn.readVLong();
- final int docCount = termsMetaIn.readVInt();
- if (version < VERSION_META_LONGS_REMOVED) {
- final int longsSize = termsMetaIn.readVInt();
- if (longsSize < 0) {
- throw new CorruptIndexException(
- "invalid longsSize for field: " + fieldInfo.name + ", longsSize=" + longsSize,
- termsMetaIn);
- }
- }
- BytesRef minTerm = readBytesRef(termsMetaIn);
- BytesRef maxTerm = readBytesRef(termsMetaIn);
- if (docCount < 0
- || docCount > state.segmentInfo.maxDoc()) { // #docs with field must be <= #docs
- throw new CorruptIndexException(
- "invalid docCount: " + docCount + " maxDoc: " + state.segmentInfo.maxDoc(),
- termsMetaIn);
- }
- if (sumDocFreq < docCount) { // #postings must be >= #docs with field
- throw new CorruptIndexException(
- "invalid sumDocFreq: " + sumDocFreq + " docCount: " + docCount, termsMetaIn);
- }
- if (sumTotalTermFreq < sumDocFreq) { // #positions must be >= #postings
- throw new CorruptIndexException(
- "invalid sumTotalTermFreq: " + sumTotalTermFreq + " sumDocFreq: " + sumDocFreq,
- termsMetaIn);
- }
- final long indexStartFP = indexMetaIn.readVLong();
- FieldReader previous =
- fieldMap.put(
- fieldInfo.name,
- new FieldReader(
- this,
- fieldInfo,
- numTerms,
- rootCode,
- sumTotalTermFreq,
- sumDocFreq,
- docCount,
- indexStartFP,
- indexMetaIn,
- indexIn,
- minTerm,
- maxTerm));
- if (previous != null) {
- throw new CorruptIndexException("duplicate field: " + fieldInfo.name, termsMetaIn);
- }
- }
- if (version >= VERSION_META_FILE) {
- indexLength = metaIn.readLong();
- termsLength = metaIn.readLong();
- }
- } catch (Throwable exception) {
- priorE = exception;
- } finally {
- if (metaIn != null) {
- CodecUtil.checkFooter(metaIn, priorE);
- } else if (priorE != null) {
- IOUtils.rethrowAlways(priorE);
- }
- }
- }
- if (version >= VERSION_META_FILE) {
- // At this point the checksum of the meta file has been verified so the lengths are likely
- // correct
- CodecUtil.retrieveChecksum(indexIn, indexLength);
- CodecUtil.retrieveChecksum(termsIn, termsLength);
- } else {
- assert indexLength == -1 : indexLength;
- assert termsLength == -1 : termsLength;
- }
- List<String> fieldList = new ArrayList<>(fieldMap.keySet());
- fieldList.sort(null);
- this.fieldMap = fieldMap;
- this.fieldList = Collections.unmodifiableList(fieldList);
- success = true;
- } finally {
- if (!success) {
- // this.close() will close in:
- IOUtils.closeWhileHandlingException(this);
- }
- }
- }
-
- private static BytesRef readBytesRef(IndexInput in) throws IOException {
- int numBytes = in.readVInt();
- if (numBytes < 0) {
- throw new CorruptIndexException("invalid bytes length: " + numBytes, in);
- }
-
- BytesRef bytes = new BytesRef();
- bytes.length = numBytes;
- bytes.bytes = new byte[numBytes];
- in.readBytes(bytes.bytes, 0, numBytes);
-
- return bytes;
- }
-
- /** Seek {@code input} to the directory offset. */
- private static void seekDir(IndexInput input) throws IOException {
- input.seek(input.length() - CodecUtil.footerLength() - 8);
- long offset = input.readLong();
- input.seek(offset);
- }
-
- // for debugging
- // private static String toHex(int v) {
- // return "0x" + Integer.toHexString(v);
- // }
-
- @Override
- public void close() throws IOException {
- try {
- IOUtils.close(indexIn, termsIn, postingsReader);
- } finally {
- // Clear so refs to terms index is GCable even if
- // app hangs onto us:
- fieldMap.clear();
- }
- }
-
- @Override
- public Iterator<String> iterator() {
- return fieldList.iterator();
- }
-
- @Override
- public Terms terms(String field) throws IOException {
- assert field != null;
- return fieldMap.get(field);
- }
-
- @Override
- public int size() {
- return fieldMap.size();
- }
-
- @Override
- public void checkIntegrity() throws IOException {
- // terms index
- CodecUtil.checksumEntireFile(indexIn);
-
- // term dictionary
- CodecUtil.checksumEntireFile(termsIn);
-
- // postings
- postingsReader.checkIntegrity();
- }
-
- @Override
- public String toString() {
- return getClass().getSimpleName()
- + "(fields="
- + fieldMap.size()
- + ",delegate="
- + postingsReader
- + ")";
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/SegmentTermsEnum.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/SegmentTermsEnum.java
deleted file mode 100644
index 224906fa5a0..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/SegmentTermsEnum.java
+++ /dev/null
@@ -1,1177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene40.blocktree;
-
-import java.io.IOException;
-import java.io.PrintStream;
-import org.apache.lucene.codecs.BlockTermState;
-import org.apache.lucene.index.BaseTermsEnum;
-import org.apache.lucene.index.ImpactsEnum;
-import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.TermState;
-import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.RamUsageEstimator;
-import org.apache.lucene.util.fst.FST;
-import org.apache.lucene.util.fst.Util;
-
-/** Iterates through terms in this field. */
-final class SegmentTermsEnum extends BaseTermsEnum {
-
- // Lazy init:
- IndexInput in;
-
- private SegmentTermsEnumFrame[] stack;
- private final SegmentTermsEnumFrame staticFrame;
- SegmentTermsEnumFrame currentFrame;
- boolean termExists;
- final FieldReader fr;
-
- private int targetBeforeCurrentLength;
-
- // static boolean DEBUG = BlockTreeTermsWriter.DEBUG;
-
- private final ByteArrayDataInput scratchReader = new ByteArrayDataInput();
-
- // What prefix of the current term was present in the index; when we only next() through the
- // index, this stays at 0. It's only set when
- // we seekCeil/Exact:
- private int validIndexPrefix;
-
- // assert only:
- private boolean eof;
-
- final BytesRefBuilder term = new BytesRefBuilder();
- private final FST.BytesReader fstReader;
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- private FST.Arc<BytesRef>[] arcs = new FST.Arc[1];
-
- public SegmentTermsEnum(FieldReader fr) throws IOException {
- this.fr = fr;
-
- // if (DEBUG) {
- // System.out.println("BTTR.init seg=" + fr.parent.segment);
- // }
- stack = new SegmentTermsEnumFrame[0];
-
- // Used to hold seek by TermState, or cached seek
- staticFrame = new SegmentTermsEnumFrame(this, -1);
-
- if (fr.index == null) {
- fstReader = null;
- } else {
- fstReader = fr.index.getBytesReader();
- }
-
- // Init w/ root block; don't use index since it may
- // not (and need not) have been loaded
- for (int arcIdx = 0; arcIdx < arcs.length; arcIdx++) {
- arcs[arcIdx] = new FST.Arc<>();
- }
-
- currentFrame = staticFrame;
- final FST.Arc<BytesRef> arc;
- if (fr.index != null) {
- arc = fr.index.getFirstArc(arcs[0]);
- // Empty string prefix must have an output in the index!
- assert arc.isFinal();
- } else {
- arc = null;
- }
- // currentFrame = pushFrame(arc, rootCode, 0);
- // currentFrame.loadBlock();
- validIndexPrefix = 0;
- // if (DEBUG) {
- // System.out.println("init frame state " + currentFrame.ord);
- // printSeekState();
- // }
-
- // System.out.println();
- // computeBlockStats().print(System.out);
- }
-
- // Not private to avoid synthetic access$NNN methods
- void initIndexInput() {
- if (this.in == null) {
- this.in = fr.parent.termsIn.clone();
- }
- }
-
- /** Runs next() through the entire terms dict, computing aggregate statistics. */
- public Stats computeBlockStats() throws IOException {
-
- Stats stats = new Stats(fr.parent.segment, fr.fieldInfo.name);
- if (fr.index != null) {
- stats.indexNumBytes = fr.index.ramBytesUsed();
- }
-
- currentFrame = staticFrame;
- FST.Arc<BytesRef> arc;
- if (fr.index != null) {
- arc = fr.index.getFirstArc(arcs[0]);
- // Empty string prefix must have an output in the index!
- assert arc.isFinal();
- } else {
- arc = null;
- }
-
- // Empty string prefix must have an output in the
- // index!
- currentFrame = pushFrame(arc, fr.rootCode, 0);
- currentFrame.fpOrig = currentFrame.fp;
- currentFrame.loadBlock();
- validIndexPrefix = 0;
-
- stats.startBlock(currentFrame, !currentFrame.isLastInFloor);
-
- allTerms:
- while (true) {
-
- // Pop finished blocks
- while (currentFrame.nextEnt == currentFrame.entCount) {
- stats.endBlock(currentFrame);
- if (!currentFrame.isLastInFloor) {
- // Advance to next floor block
- currentFrame.loadNextFloorBlock();
- stats.startBlock(currentFrame, true);
- break;
- } else {
- if (currentFrame.ord == 0) {
- break allTerms;
- }
- final long lastFP = currentFrame.fpOrig;
- currentFrame = stack[currentFrame.ord - 1];
- assert lastFP == currentFrame.lastSubFP;
- // if (DEBUG) {
- // System.out.println(" reset validIndexPrefix=" + validIndexPrefix);
- // }
- }
- }
-
- while (true) {
- if (currentFrame.next()) {
- // Push to new block:
- currentFrame = pushFrame(null, currentFrame.lastSubFP, term.length());
- currentFrame.fpOrig = currentFrame.fp;
- // This is a "next" frame -- even if it's
- // floor'd we must pretend it isn't so we don't
- // try to scan to the right floor frame:
- currentFrame.loadBlock();
- stats.startBlock(currentFrame, !currentFrame.isLastInFloor);
- } else {
- stats.term(term.get());
- break;
- }
- }
- }
-
- stats.finish();
-
- // Put root frame back:
- currentFrame = staticFrame;
- if (fr.index != null) {
- arc = fr.index.getFirstArc(arcs[0]);
- // Empty string prefix must have an output in the index!
- assert arc.isFinal();
- } else {
- arc = null;
- }
- currentFrame = pushFrame(arc, fr.rootCode, 0);
- currentFrame.rewind();
- currentFrame.loadBlock();
- validIndexPrefix = 0;
- term.clear();
-
- return stats;
- }
-
- private SegmentTermsEnumFrame getFrame(int ord) throws IOException {
- if (ord >= stack.length) {
- final SegmentTermsEnumFrame[] next =
- new SegmentTermsEnumFrame
- [ArrayUtil.oversize(1 + ord, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
- System.arraycopy(stack, 0, next, 0, stack.length);
- for (int stackOrd = stack.length; stackOrd < next.length; stackOrd++) {
- next[stackOrd] = new SegmentTermsEnumFrame(this, stackOrd);
- }
- stack = next;
- }
- assert stack[ord].ord == ord;
- return stack[ord];
- }
-
- private FST.Arc<BytesRef> getArc(int ord) {
- if (ord >= arcs.length) {
- @SuppressWarnings({"rawtypes", "unchecked"})
- final FST.Arc<BytesRef>[] next =
- new FST.Arc[ArrayUtil.oversize(1 + ord, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
- System.arraycopy(arcs, 0, next, 0, arcs.length);
- for (int arcOrd = arcs.length; arcOrd < next.length; arcOrd++) {
- next[arcOrd] = new FST.Arc<>();
- }
- arcs = next;
- }
- return arcs[ord];
- }
-
- // Pushes a frame we seek'd to
- SegmentTermsEnumFrame pushFrame(FST.Arc<BytesRef> arc, BytesRef frameData, int length)
- throws IOException {
- scratchReader.reset(frameData.bytes, frameData.offset, frameData.length);
- final long code = scratchReader.readVLong();
- final long fpSeek = code >>> Lucene40BlockTreeTermsReader.OUTPUT_FLAGS_NUM_BITS;
- final SegmentTermsEnumFrame f = getFrame(1 + currentFrame.ord);
- f.hasTerms = (code & Lucene40BlockTreeTermsReader.OUTPUT_FLAG_HAS_TERMS) != 0;
- f.hasTermsOrig = f.hasTerms;
- f.isFloor = (code & Lucene40BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR) != 0;
- if (f.isFloor) {
- f.setFloorData(scratchReader, frameData);
- }
- pushFrame(arc, fpSeek, length);
-
- return f;
- }
-
- // Pushes next'd frame or seek'd frame; we later
- // lazy-load the frame only when needed
- SegmentTermsEnumFrame pushFrame(FST.Arc<BytesRef> arc, long fp, int length) throws IOException {
- final SegmentTermsEnumFrame f = getFrame(1 + currentFrame.ord);
- f.arc = arc;
- if (f.fpOrig == fp && f.nextEnt != -1) {
- // if (DEBUG) System.out.println(" push reused frame ord=" + f.ord + " fp=" + f.fp +
- // " isFloor?=" + f.isFloor + " hasTerms=" + f.hasTerms + " pref=" + term + " nextEnt=" +
- // f.nextEnt + " targetBeforeCurrentLength=" + targetBeforeCurrentLength + " term.length=" +
- // term.length + " vs prefix=" + f.prefix);
- // if (f.prefix > targetBeforeCurrentLength) {
- if (f.ord > targetBeforeCurrentLength) {
- f.rewind();
- } else {
- // if (DEBUG) {
- // System.out.println(" skip rewind!");
- // }
- }
- assert length == f.prefix;
- } else {
- f.nextEnt = -1;
- f.prefix = length;
- f.state.termBlockOrd = 0;
- f.fpOrig = f.fp = fp;
- f.lastSubFP = -1;
- // if (DEBUG) {
- // final int sav = term.length;
- // term.length = length;
- // System.out.println(" push new frame ord=" + f.ord + " fp=" + f.fp + " hasTerms=" +
- // f.hasTerms + " isFloor=" + f.isFloor + " pref=" + ToStringUtils.bytesRefToString(term));
- // term.length = sav;
- // }
- }
-
- return f;
- }
-
- // asserts only
- private boolean clearEOF() {
- eof = false;
- return true;
- }
-
- // asserts only
- private boolean setEOF() {
- eof = true;
- return true;
- }
-
- @Override
- public boolean seekExact(BytesRef target) throws IOException {
-
- if (fr.index == null) {
- throw new IllegalStateException("terms index was not loaded");
- }
-
- if (fr.size() > 0 && (target.compareTo(fr.getMin()) < 0 || target.compareTo(fr.getMax()) > 0)) {
- return false;
- }
-
- term.grow(1 + target.length);
-
- assert clearEOF();
-
- // if (DEBUG) {
- // System.out.println("\nBTTR.seekExact seg=" + fr.parent.segment + " target=" +
- // fr.fieldInfo.name + ":" + ToStringUtils.bytesRefToString(target) + " current=" +
- // ToStringUtils.bytesRefToString(term) + " (exists?=" + termExists +
- // ") validIndexPrefix=" + validIndexPrefix);
- // printSeekState(System.out);
- // }
-
- FST.Arc<BytesRef> arc;
- int targetUpto;
- BytesRef output;
-
- targetBeforeCurrentLength = currentFrame.ord;
-
- if (currentFrame != staticFrame) {
-
- // We are already seek'd; find the common
- // prefix of new seek term vs current term and
- // re-use the corresponding seek state. For
- // example, if app first seeks to foobar, then
- // seeks to foobaz, we can re-use the seek state
- // for the first 5 bytes.
-
- // if (DEBUG) {
- // System.out.println(" re-use current seek state validIndexPrefix=" + validIndexPrefix);
- // }
-
- arc = arcs[0];
- assert arc.isFinal();
- output = arc.output();
- targetUpto = 0;
-
- SegmentTermsEnumFrame lastFrame = stack[0];
- assert validIndexPrefix <= term.length();
-
- final int targetLimit = Math.min(target.length, validIndexPrefix);
-
- int cmp = 0;
-
- // TODO: reverse vLong byte order for better FST
- // prefix output sharing
-
- // First compare up to valid seek frames:
- while (targetUpto < targetLimit) {
- cmp = (term.byteAt(targetUpto) & 0xFF) - (target.bytes[target.offset + targetUpto] & 0xFF);
- // if (DEBUG) {
- // System.out.println(" cycle targetUpto=" + targetUpto + " (vs limit=" + targetLimit
- // + ") cmp=" + cmp + " (targetLabel=" + (char) (target.bytes[target.offset + targetUpto]) +
- // " vs termLabel=" + (char) (term.bytes[targetUpto]) + ")" + " arc.output=" + arc.output
- // + " output=" + output);
- // }
- if (cmp != 0) {
- break;
- }
- arc = arcs[1 + targetUpto];
- assert arc.label() == (target.bytes[target.offset + targetUpto] & 0xFF)
- : "arc.label="
- + (char) arc.label()
- + " targetLabel="
- + (char) (target.bytes[target.offset + targetUpto] & 0xFF);
- if (arc.output() != Lucene40BlockTreeTermsReader.NO_OUTPUT) {
- output = Lucene40BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.output());
- }
- if (arc.isFinal()) {
- lastFrame = stack[1 + lastFrame.ord];
- }
- targetUpto++;
- }
-
- if (cmp == 0) {
- final int targetUptoMid = targetUpto;
-
- // Second compare the rest of the term, but
- // don't save arc/output/frame; we only do this
- // to find out if the target term is before,
- // equal or after the current term
- final int targetLimit2 = Math.min(target.length, term.length());
- while (targetUpto < targetLimit2) {
- cmp =
- (term.byteAt(targetUpto) & 0xFF) - (target.bytes[target.offset + targetUpto] & 0xFF);
- // if (DEBUG) {
- // System.out.println(" cycle2 targetUpto=" + targetUpto + " (vs limit=" +
- // targetLimit + ") cmp=" + cmp + " (targetLabel=" + (char) (target.bytes[target.offset +
- // targetUpto]) + " vs termLabel=" + (char) (term.bytes[targetUpto]) + ")");
- // }
- if (cmp != 0) {
- break;
- }
- targetUpto++;
- }
-
- if (cmp == 0) {
- cmp = term.length() - target.length;
- }
- targetUpto = targetUptoMid;
- }
-
- if (cmp < 0) {
- // Common case: target term is after current
- // term, ie, app is seeking multiple terms
- // in sorted order
- // if (DEBUG) {
- // System.out.println(" target is after current (shares prefixLen=" + targetUpto + ");
- // frame.ord=" + lastFrame.ord);
- // }
- currentFrame = lastFrame;
-
- } else if (cmp > 0) {
- // Uncommon case: target term
- // is before current term; this means we can
- // keep the currentFrame but we must rewind it
- // (so we scan from the start)
- targetBeforeCurrentLength = lastFrame.ord;
- // if (DEBUG) {
- // System.out.println(" target is before current (shares prefixLen=" + targetUpto + ");
- // rewind frame ord=" + lastFrame.ord);
- // }
- currentFrame = lastFrame;
- currentFrame.rewind();
- } else {
- // Target is exactly the same as current term
- assert term.length() == target.length;
- if (termExists) {
- // if (DEBUG) {
- // System.out.println(" target is same as current; return true");
- // }
- return true;
- } else {
- // if (DEBUG) {
- // System.out.println(" target is same as current but term doesn't exist");
- // }
- }
- // validIndexPrefix = currentFrame.depth;
- // term.length = target.length;
- // return termExists;
- }
-
- } else {
-
- targetBeforeCurrentLength = -1;
- arc = fr.index.getFirstArc(arcs[0]);
-
- // Empty string prefix must have an output (block) in the index!
- assert arc.isFinal();
- assert arc.output() != null;
-
- // if (DEBUG) {
- // System.out.println(" no seek state; push root frame");
- // }
-
- output = arc.output();
-
- currentFrame = staticFrame;
-
- // term.length = 0;
- targetUpto = 0;
- currentFrame =
- pushFrame(
- arc, Lucene40BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.nextFinalOutput()), 0);
- }
-
- // if (DEBUG) {
- // System.out.println(" start index loop targetUpto=" + targetUpto + " output=" + output +
- // " currentFrame.ord=" + currentFrame.ord + " targetBeforeCurrentLength=" +
- // targetBeforeCurrentLength);
- // }
-
- // We are done sharing the common prefix with the incoming target and where we are currently
- // seek'd; now continue walking the index:
- while (targetUpto < target.length) {
-
- final int targetLabel = target.bytes[target.offset + targetUpto] & 0xFF;
-
- final FST.Arc<BytesRef> nextArc =
- fr.index.findTargetArc(targetLabel, arc, getArc(1 + targetUpto), fstReader);
-
- if (nextArc == null) {
-
- // Index is exhausted
- // if (DEBUG) {
- // System.out.println(" index: index exhausted label=" + ((char) targetLabel) + " " +
- // toHex(targetLabel));
- // }
-
- validIndexPrefix = currentFrame.prefix;
- // validIndexPrefix = targetUpto;
-
- currentFrame.scanToFloorFrame(target);
-
- if (!currentFrame.hasTerms) {
- termExists = false;
- term.setByteAt(targetUpto, (byte) targetLabel);
- term.setLength(1 + targetUpto);
- // if (DEBUG) {
- // System.out.println(" FAST NOT_FOUND term=" + ToStringUtils.bytesRefToString(term));
- // }
- return false;
- }
-
- currentFrame.loadBlock();
-
- final SeekStatus result = currentFrame.scanToTerm(target, true);
- if (result == SeekStatus.FOUND) {
- // if (DEBUG) {
- // System.out.println(" return FOUND term=" + term.utf8ToString() + " " + term);
- // }
- return true;
- } else {
- // if (DEBUG) {
- // System.out.println(" got " + result + "; return NOT_FOUND term=" +
- // ToStringUtils.bytesRefToString(term));
- // }
- return false;
- }
- } else {
- // Follow this arc
- arc = nextArc;
- term.setByteAt(targetUpto, (byte) targetLabel);
- // Aggregate output as we go:
- assert arc.output() != null;
- if (arc.output() != Lucene40BlockTreeTermsReader.NO_OUTPUT) {
- output = Lucene40BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.output());
- }
-
- // if (DEBUG) {
- // System.out.println(" index: follow label=" + toHex(target.bytes[target.offset +
- // targetUpto]&0xff) + " arc.output=" + arc.output + " arc.nfo=" + arc.nextFinalOutput);
- // }
- targetUpto++;
-
- if (arc.isFinal()) {
- // if (DEBUG) System.out.println(" arc is final!");
- currentFrame =
- pushFrame(
- arc,
- Lucene40BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.nextFinalOutput()),
- targetUpto);
- // if (DEBUG) System.out.println(" curFrame.ord=" + currentFrame.ord + " hasTerms=" +
- // currentFrame.hasTerms);
- }
- }
- }
-
- // validIndexPrefix = targetUpto;
- validIndexPrefix = currentFrame.prefix;
-
- currentFrame.scanToFloorFrame(target);
-
- // Target term is entirely contained in the index:
- if (!currentFrame.hasTerms) {
- termExists = false;
- term.setLength(targetUpto);
- // if (DEBUG) {
- // System.out.println(" FAST NOT_FOUND term=" + ToStringUtils.bytesRefToString(term));
- // }
- return false;
- }
-
- currentFrame.loadBlock();
-
- final SeekStatus result = currentFrame.scanToTerm(target, true);
- if (result == SeekStatus.FOUND) {
- // if (DEBUG) {
- // System.out.println(" return FOUND term=" + term.utf8ToString() + " " + term);
- // }
- return true;
- } else {
- // if (DEBUG) {
- // System.out.println(" got result " + result + "; return NOT_FOUND term=" +
- // term.utf8ToString());
- // }
-
- return false;
- }
- }
-
- @Override
- public SeekStatus seekCeil(BytesRef target) throws IOException {
-
- if (fr.index == null) {
- throw new IllegalStateException("terms index was not loaded");
- }
-
- term.grow(1 + target.length);
-
- assert clearEOF();
-
- // if (DEBUG) {
- // System.out.println("\nBTTR.seekCeil seg=" + fr.parent.segment + " target=" +
- // fr.fieldInfo.name + ":" + ToStringUtils.bytesRefToString(target) +
- // " current=" + ToStringUtils.bytesRefToString(term)
- // + " (exists?=" + termExists + ") validIndexPrefix= " + validIndexPrefix);
- // printSeekState(System.out);
- // }
-
- FST.Arc<BytesRef> arc;
- int targetUpto;
- BytesRef output;
-
- targetBeforeCurrentLength = currentFrame.ord;
-
- if (currentFrame != staticFrame) {
-
- // We are already seek'd; find the common
- // prefix of new seek term vs current term and
- // re-use the corresponding seek state. For
- // example, if app first seeks to foobar, then
- // seeks to foobaz, we can re-use the seek state
- // for the first 5 bytes.
-
- // if (DEBUG) {
- // System.out.println(" re-use current seek state validIndexPrefix=" + validIndexPrefix);
- // }
-
- arc = arcs[0];
- assert arc.isFinal();
- output = arc.output();
- targetUpto = 0;
-
- SegmentTermsEnumFrame lastFrame = stack[0];
- assert validIndexPrefix <= term.length();
-
- final int targetLimit = Math.min(target.length, validIndexPrefix);
-
- int cmp = 0;
-
- // TODO: we should write our vLong backwards (MSB
- // first) to get better sharing from the FST
-
- // First compare up to valid seek frames:
- while (targetUpto < targetLimit) {
- cmp = (term.byteAt(targetUpto) & 0xFF) - (target.bytes[target.offset + targetUpto] & 0xFF);
- // if (DEBUG) {
- // System.out.println(" cycle targetUpto=" + targetUpto + " (vs limit=" + targetLimit +
- // ") cmp=" + cmp + " (targetLabel=" + (char) (target.bytes[target.offset + targetUpto]) +
- // " vs termLabel=" + (char) (term.byteAt(targetUpto)) + ")" + " arc.output=" + arc.output
- // + " output=" + output);
- // }
- if (cmp != 0) {
- break;
- }
- arc = arcs[1 + targetUpto];
- assert arc.label() == (target.bytes[target.offset + targetUpto] & 0xFF)
- : "arc.label="
- + (char) arc.label()
- + " targetLabel="
- + (char) (target.bytes[target.offset + targetUpto] & 0xFF);
- // TODO: we could save the outputs in local
- // byte[][] instead of making new objs ever
- // seek; but, often the FST doesn't have any
- // shared bytes (but this could change if we
- // reverse vLong byte order)
- if (arc.output() != Lucene40BlockTreeTermsReader.NO_OUTPUT) {
- output = Lucene40BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.output());
- }
- if (arc.isFinal()) {
- lastFrame = stack[1 + lastFrame.ord];
- }
- targetUpto++;
- }
-
- if (cmp == 0) {
- final int targetUptoMid = targetUpto;
- // Second compare the rest of the term, but
- // don't save arc/output/frame:
- final int targetLimit2 = Math.min(target.length, term.length());
- while (targetUpto < targetLimit2) {
- cmp =
- (term.byteAt(targetUpto) & 0xFF) - (target.bytes[target.offset + targetUpto] & 0xFF);
- // if (DEBUG) {
- // System.out.println(" cycle2 targetUpto=" + targetUpto + " (vs limit=" + targetLimit
- // + ") cmp=" + cmp + " (targetLabel=" + (char) (target.bytes[target.offset + targetUpto])
- // + " vs termLabel=" + (char) (term.byteAt(targetUpto)) + ")");
- // }
- if (cmp != 0) {
- break;
- }
- targetUpto++;
- }
-
- if (cmp == 0) {
- cmp = term.length() - target.length;
- }
- targetUpto = targetUptoMid;
- }
-
- if (cmp < 0) {
- // Common case: target term is after current
- // term, ie, app is seeking multiple terms
- // in sorted order
- // if (DEBUG) {
- // System.out.println(" target is after current (shares prefixLen=" + targetUpto + ");
- // clear frame.scanned ord=" + lastFrame.ord);
- // }
- currentFrame = lastFrame;
-
- } else if (cmp > 0) {
- // Uncommon case: target term
- // is before current term; this means we can
- // keep the currentFrame but we must rewind it
- // (so we scan from the start)
- targetBeforeCurrentLength = 0;
- // if (DEBUG) {
- // System.out.println(" target is before current (shares prefixLen=" + targetUpto + ");
- // rewind frame ord=" + lastFrame.ord);
- // }
- currentFrame = lastFrame;
- currentFrame.rewind();
- } else {
- // Target is exactly the same as current term
- assert term.length() == target.length;
- if (termExists) {
- // if (DEBUG) {
- // System.out.println(" target is same as current; return FOUND");
- // }
- return SeekStatus.FOUND;
- } else {
- // if (DEBUG) {
- // System.out.println(" target is same as current but term doesn't exist");
- // }
- }
- }
-
- } else {
-
- targetBeforeCurrentLength = -1;
- arc = fr.index.getFirstArc(arcs[0]);
-
- // Empty string prefix must have an output (block) in the index!
- assert arc.isFinal();
- assert arc.output() != null;
-
- // if (DEBUG) {
- // System.out.println(" no seek state; push root frame");
- // }
-
- output = arc.output();
-
- currentFrame = staticFrame;
-
- // term.length = 0;
- targetUpto = 0;
- currentFrame =
- pushFrame(
- arc, Lucene40BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.nextFinalOutput()), 0);
- }
-
- // if (DEBUG) {
- // System.out.println(" start index loop targetUpto=" + targetUpto + " output=" + output +
- // " currentFrame.ord+1=" + currentFrame.ord + " targetBeforeCurrentLength=" +
- // targetBeforeCurrentLength);
- // }
-
- // We are done sharing the common prefix with the incoming target and where we are currently
- // seek'd; now continue walking the index:
- while (targetUpto < target.length) {
-
- final int targetLabel = target.bytes[target.offset + targetUpto] & 0xFF;
-
- final FST.Arc<BytesRef> nextArc =
- fr.index.findTargetArc(targetLabel, arc, getArc(1 + targetUpto), fstReader);
-
- if (nextArc == null) {
-
- // Index is exhausted
- // if (DEBUG) {
- // System.out.println(" index: index exhausted label=" + ((char) targetLabel) + " " +
- // targetLabel);
- // }
-
- validIndexPrefix = currentFrame.prefix;
- // validIndexPrefix = targetUpto;
-
- currentFrame.scanToFloorFrame(target);
-
- currentFrame.loadBlock();
-
- // if (DEBUG) System.out.println(" now scanToTerm");
- final SeekStatus result = currentFrame.scanToTerm(target, false);
- if (result == SeekStatus.END) {
- term.copyBytes(target);
- termExists = false;
-
- if (next() != null) {
- // if (DEBUG) {
- // System.out.println(" return NOT_FOUND term=" +
- // ToStringUtils.bytesRefToString(term));
- // }
- return SeekStatus.NOT_FOUND;
- } else {
- // if (DEBUG) {
- // System.out.println(" return END");
- // }
- return SeekStatus.END;
- }
- } else {
- // if (DEBUG) {
- // System.out.println(" return " + result + " term=" +
- // ToStringUtils.bytesRefToString(term));
- // }
- return result;
- }
- } else {
- // Follow this arc
- term.setByteAt(targetUpto, (byte) targetLabel);
- arc = nextArc;
- // Aggregate output as we go:
- assert arc.output() != null;
- if (arc.output() != Lucene40BlockTreeTermsReader.NO_OUTPUT) {
- output = Lucene40BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.output());
- }
-
- // if (DEBUG) {
- // System.out.println(" index: follow label=" + (target.bytes[target.offset +
- // targetUpto]&0xff) + " arc.output=" + arc.output + " arc.nfo=" + arc.nextFinalOutput);
- // }
- targetUpto++;
-
- if (arc.isFinal()) {
- // if (DEBUG) System.out.println(" arc is final!");
- currentFrame =
- pushFrame(
- arc,
- Lucene40BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.nextFinalOutput()),
- targetUpto);
- // if (DEBUG) System.out.println(" curFrame.ord=" + currentFrame.ord + " hasTerms=" +
- // currentFrame.hasTerms);
- }
- }
- }
-
- // validIndexPrefix = targetUpto;
- validIndexPrefix = currentFrame.prefix;
-
- currentFrame.scanToFloorFrame(target);
-
- currentFrame.loadBlock();
-
- final SeekStatus result = currentFrame.scanToTerm(target, false);
-
- if (result == SeekStatus.END) {
- term.copyBytes(target);
- termExists = false;
- if (next() != null) {
- // if (DEBUG) {
- // System.out.println(" return NOT_FOUND term=" + term.get().utf8ToString() + " " + term);
- // }
- return SeekStatus.NOT_FOUND;
- } else {
- // if (DEBUG) {
- // System.out.println(" return END");
- // }
- return SeekStatus.END;
- }
- } else {
- return result;
- }
- }
-
- @SuppressWarnings("unused")
- private void printSeekState(PrintStream out) throws IOException {
- if (currentFrame == staticFrame) {
- out.println(" no prior seek");
- } else {
- out.println(" prior seek state:");
- int ord = 0;
- boolean isSeekFrame = true;
- while (true) {
- SegmentTermsEnumFrame f = getFrame(ord);
- assert f != null;
- final BytesRef prefix = new BytesRef(term.get().bytes, 0, f.prefix);
- if (f.nextEnt == -1) {
- out.println(
- " frame "
- + (isSeekFrame ? "(seek)" : "(next)")
- + " ord="
- + ord
- + " fp="
- + f.fp
- + (f.isFloor ? (" (fpOrig=" + f.fpOrig + ")") : "")
- + " prefixLen="
- + f.prefix
- + " prefix="
- + prefix
- + (f.nextEnt == -1 ? "" : (" (of " + f.entCount + ")"))
- + " hasTerms="
- + f.hasTerms
- + " isFloor="
- + f.isFloor
- + " code="
- + ((f.fp << Lucene40BlockTreeTermsReader.OUTPUT_FLAGS_NUM_BITS)
- + (f.hasTerms ? Lucene40BlockTreeTermsReader.OUTPUT_FLAG_HAS_TERMS : 0)
- + (f.isFloor ? Lucene40BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR : 0))
- + " isLastInFloor="
- + f.isLastInFloor
- + " mdUpto="
- + f.metaDataUpto
- + " tbOrd="
- + f.getTermBlockOrd());
- } else {
- out.println(
- " frame "
- + (isSeekFrame ? "(seek, loaded)" : "(next, loaded)")
- + " ord="
- + ord
- + " fp="
- + f.fp
- + (f.isFloor ? (" (fpOrig=" + f.fpOrig + ")") : "")
- + " prefixLen="
- + f.prefix
- + " prefix="
- + prefix
- + " nextEnt="
- + f.nextEnt
- + (f.nextEnt == -1 ? "" : (" (of " + f.entCount + ")"))
- + " hasTerms="
- + f.hasTerms
- + " isFloor="
- + f.isFloor
- + " code="
- + ((f.fp << Lucene40BlockTreeTermsReader.OUTPUT_FLAGS_NUM_BITS)
- + (f.hasTerms ? Lucene40BlockTreeTermsReader.OUTPUT_FLAG_HAS_TERMS : 0)
- + (f.isFloor ? Lucene40BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR : 0))
- + " lastSubFP="
- + f.lastSubFP
- + " isLastInFloor="
- + f.isLastInFloor
- + " mdUpto="
- + f.metaDataUpto
- + " tbOrd="
- + f.getTermBlockOrd());
- }
- if (fr.index != null) {
- assert !isSeekFrame || f.arc != null : "isSeekFrame=" + isSeekFrame + " f.arc=" + f.arc;
- if (f.prefix > 0 && isSeekFrame && f.arc.label() != (term.byteAt(f.prefix - 1) & 0xFF)) {
- out.println(
- " broken seek state: arc.label="
- + (char) f.arc.label()
- + " vs term byte="
- + (char) (term.byteAt(f.prefix - 1) & 0xFF));
- throw new RuntimeException("seek state is broken");
- }
- BytesRef output = Util.get(fr.index, prefix);
- if (output == null) {
- out.println(" broken seek state: prefix is not final in index");
- throw new RuntimeException("seek state is broken");
- } else if (isSeekFrame && !f.isFloor) {
- final ByteArrayDataInput reader =
- new ByteArrayDataInput(output.bytes, output.offset, output.length);
- final long codeOrig = reader.readVLong();
- final long code =
- (f.fp << Lucene40BlockTreeTermsReader.OUTPUT_FLAGS_NUM_BITS)
- | (f.hasTerms ? Lucene40BlockTreeTermsReader.OUTPUT_FLAG_HAS_TERMS : 0)
- | (f.isFloor ? Lucene40BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR : 0);
- if (codeOrig != code) {
- out.println(
- " broken seek state: output code="
- + codeOrig
- + " doesn't match frame code="
- + code);
- throw new RuntimeException("seek state is broken");
- }
- }
- }
- if (f == currentFrame) {
- break;
- }
- if (f.prefix == validIndexPrefix) {
- isSeekFrame = false;
- }
- ord++;
- }
- }
- }
-
- /* Decodes only the term bytes of the next term. If caller then asks for
- metadata, ie docFreq, totalTermFreq or pulls a D/&PEnum, we then (lazily)
- decode all metadata up to the current term. */
- @Override
- public BytesRef next() throws IOException {
- if (in == null) {
- // Fresh TermsEnum; seek to first term:
- final FST.Arc<BytesRef> arc;
- if (fr.index != null) {
- arc = fr.index.getFirstArc(arcs[0]);
- // Empty string prefix must have an output in the index!
- assert arc.isFinal();
- } else {
- arc = null;
- }
- currentFrame = pushFrame(arc, fr.rootCode, 0);
- currentFrame.loadBlock();
- }
-
- targetBeforeCurrentLength = currentFrame.ord;
-
- assert !eof;
- // if (DEBUG) {
- // System.out.println("\nBTTR.next seg=" + fr.parent.segment + " term=" +
- // ToStringUtils.bytesRefToString(term) + " termExists?=" + termExists + " field=" +
- // fr.fieldInfo.name + " termBlockOrd=" + currentFrame.state.termBlockOrd +
- // " validIndexPrefix=" + validIndexPrefix);
- // printSeekState(System.out);
- // }
-
- if (currentFrame == staticFrame) {
- // If seek was previously called and the term was
- // cached, or seek(TermState) was called, usually
- // caller is just going to pull a D/&PEnum or get
- // docFreq, etc. But, if they then call next(),
- // this method catches up all internal state so next()
- // works properly:
- // if (DEBUG) System.out.println(" re-seek to pending term=" + term.utf8ToString() + " " +
- // term);
- final boolean result = seekExact(term.get());
- assert result;
- }
-
- // Pop finished blocks
- while (currentFrame.nextEnt == currentFrame.entCount) {
- if (!currentFrame.isLastInFloor) {
- // Advance to next floor block
- currentFrame.loadNextFloorBlock();
- break;
- } else {
- // if (DEBUG) System.out.println(" pop frame");
- if (currentFrame.ord == 0) {
- // if (DEBUG) System.out.println(" return null");
- assert setEOF();
- term.clear();
- validIndexPrefix = 0;
- currentFrame.rewind();
- termExists = false;
- return null;
- }
- final long lastFP = currentFrame.fpOrig;
- currentFrame = stack[currentFrame.ord - 1];
-
- if (currentFrame.nextEnt == -1 || currentFrame.lastSubFP != lastFP) {
- // We popped into a frame that's not loaded
- // yet or not scan'd to the right entry
- currentFrame.scanToFloorFrame(term.get());
- currentFrame.loadBlock();
- currentFrame.scanToSubBlock(lastFP);
- }
-
- // Note that the seek state (last seek) has been
- // invalidated beyond this depth
- validIndexPrefix = Math.min(validIndexPrefix, currentFrame.prefix);
- // if (DEBUG) {
- // System.out.println(" reset validIndexPrefix=" + validIndexPrefix);
- // }
- }
- }
-
- while (true) {
- if (currentFrame.next()) {
- // Push to new block:
- // if (DEBUG) System.out.println(" push frame");
- currentFrame = pushFrame(null, currentFrame.lastSubFP, term.length());
- // This is a "next" frame -- even if it's
- // floor'd we must pretend it isn't so we don't
- // try to scan to the right floor frame:
- currentFrame.loadBlock();
- } else {
- // if (DEBUG) System.out.println(" return term=" + ToStringUtils.bytesRefToString(term) +
- // " currentFrame.ord=" + currentFrame.ord);
- return term.get();
- }
- }
- }
-
- @Override
- public BytesRef term() {
- assert !eof;
- return term.get();
- }
-
- @Override
- public int docFreq() throws IOException {
- assert !eof;
- // if (DEBUG) System.out.println("BTR.docFreq");
- currentFrame.decodeMetaData();
- // if (DEBUG) System.out.println(" return " + currentFrame.state.docFreq);
- return currentFrame.state.docFreq;
- }
-
- @Override
- public long totalTermFreq() throws IOException {
- assert !eof;
- currentFrame.decodeMetaData();
- return currentFrame.state.totalTermFreq;
- }
-
- @Override
- public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException {
- assert !eof;
- // if (DEBUG) {
- // System.out.println("BTTR.docs seg=" + segment);
- // }
- currentFrame.decodeMetaData();
- // if (DEBUG) {
- // System.out.println(" state=" + currentFrame.state);
- // }
- return fr.parent.postingsReader.postings(fr.fieldInfo, currentFrame.state, reuse, flags);
- }
-
- @Override
- public ImpactsEnum impacts(int flags) throws IOException {
- assert !eof;
- // if (DEBUG) {
- // System.out.println("BTTR.docs seg=" + segment);
- // }
- currentFrame.decodeMetaData();
- // if (DEBUG) {
- // System.out.println(" state=" + currentFrame.state);
- // }
- return fr.parent.postingsReader.impacts(fr.fieldInfo, currentFrame.state, flags);
- }
-
- @Override
- public void seekExact(BytesRef target, TermState otherState) {
- // if (DEBUG) {
- // System.out.println("BTTR.seekExact termState seg=" + segment + " target=" +
- // target.utf8ToString() + " " + target + " state=" + otherState);
- // }
- assert clearEOF();
- if (target.compareTo(term.get()) != 0 || !termExists) {
- assert otherState != null && otherState instanceof BlockTermState;
- currentFrame = staticFrame;
- currentFrame.state.copyFrom(otherState);
- term.copyBytes(target);
- currentFrame.metaDataUpto = currentFrame.getTermBlockOrd();
- assert currentFrame.metaDataUpto > 0;
- validIndexPrefix = 0;
- } else {
- // if (DEBUG) {
- // System.out.println(" skip seek: already on target state=" + currentFrame.state);
- // }
- }
- }
-
- @Override
- public TermState termState() throws IOException {
- assert !eof;
- currentFrame.decodeMetaData();
- TermState ts = currentFrame.state.clone();
- // if (DEBUG) System.out.println("BTTR.termState seg=" + segment + " state=" + ts);
- return ts;
- }
-
- @Override
- public void seekExact(long ord) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public long ord() {
- throw new UnsupportedOperationException();
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/SegmentTermsEnumFrame.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/SegmentTermsEnumFrame.java
deleted file mode 100644
index 3003151bbe7..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/SegmentTermsEnumFrame.java
+++ /dev/null
@@ -1,794 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene40.blocktree;
-
-import java.io.IOException;
-import java.util.Arrays;
-import org.apache.lucene.codecs.BlockTermState;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.TermsEnum.SeekStatus;
-import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.fst.FST;
-
-final class SegmentTermsEnumFrame {
- // Our index in stack[]:
- final int ord;
-
- boolean hasTerms;
- boolean hasTermsOrig;
- boolean isFloor;
-
- FST.Arc<BytesRef> arc;
-
- // static boolean DEBUG = BlockTreeTermsWriter.DEBUG;
-
- // File pointer where this block was loaded from
- long fp;
- long fpOrig;
- long fpEnd;
- long totalSuffixBytes; // for stats
-
- byte[] suffixBytes = new byte[128];
- final ByteArrayDataInput suffixesReader = new ByteArrayDataInput();
-
- byte[] suffixLengthBytes;
- final ByteArrayDataInput suffixLengthsReader;
-
- byte[] statBytes = new byte[64];
- int statsSingletonRunLength = 0;
- final ByteArrayDataInput statsReader = new ByteArrayDataInput();
-
- byte[] floorData = new byte[32];
- final ByteArrayDataInput floorDataReader = new ByteArrayDataInput();
-
- // Length of prefix shared by all terms in this block
- int prefix;
-
- // Number of entries (term or sub-block) in this block
- int entCount;
-
- // Which term we will next read, or -1 if the block
- // isn't loaded yet
- int nextEnt;
-
- // True if this block is either not a floor block,
- // or, it's the last sub-block of a floor block
- boolean isLastInFloor;
-
- // True if all entries are terms
- boolean isLeafBlock;
-
- long lastSubFP;
-
- int nextFloorLabel;
- int numFollowFloorBlocks;
-
- // Next term to decode metaData; we decode metaData
- // lazily so that scanning to find the matching term is
- // fast and only if you find a match and app wants the
- // stats or docs/positions enums, will we decode the
- // metaData
- int metaDataUpto;
-
- final BlockTermState state;
-
- // metadata buffer
- byte[] bytes = new byte[32];
- final ByteArrayDataInput bytesReader = new ByteArrayDataInput();
-
- private final SegmentTermsEnum ste;
- private final int version;
-
- public SegmentTermsEnumFrame(SegmentTermsEnum ste, int ord) throws IOException {
- this.ste = ste;
- this.ord = ord;
- this.state = ste.fr.parent.postingsReader.newTermState();
- this.state.totalTermFreq = -1;
- this.version = ste.fr.parent.version;
- if (version >= Lucene40BlockTreeTermsReader.VERSION_COMPRESSED_SUFFIXES) {
- suffixLengthBytes = new byte[32];
- suffixLengthsReader = new ByteArrayDataInput();
- } else {
- suffixLengthBytes = null;
- suffixLengthsReader = suffixesReader;
- }
- }
-
- public void setFloorData(ByteArrayDataInput in, BytesRef source) {
- final int numBytes = source.length - (in.getPosition() - source.offset);
- if (numBytes > floorData.length) {
- floorData = new byte[ArrayUtil.oversize(numBytes, 1)];
- }
- System.arraycopy(source.bytes, source.offset + in.getPosition(), floorData, 0, numBytes);
- floorDataReader.reset(floorData, 0, numBytes);
- numFollowFloorBlocks = floorDataReader.readVInt();
- nextFloorLabel = floorDataReader.readByte() & 0xff;
- // if (DEBUG) {
- // System.out.println(" setFloorData fpOrig=" + fpOrig + " bytes=" + new
- // BytesRef(source.bytes, source.offset + in.getPosition(), numBytes) + " numFollowFloorBlocks="
- // + numFollowFloorBlocks + " nextFloorLabel=" + toHex(nextFloorLabel));
- // }
- }
-
- public int getTermBlockOrd() {
- return isLeafBlock ? nextEnt : state.termBlockOrd;
- }
-
- void loadNextFloorBlock() throws IOException {
- // if (DEBUG) {
- // System.out.println(" loadNextFloorBlock fp=" + fp + " fpEnd=" + fpEnd);
- // }
- assert arc == null || isFloor : "arc=" + arc + " isFloor=" + isFloor;
- fp = fpEnd;
- nextEnt = -1;
- loadBlock();
- }
-
- /* Does initial decode of next block of terms; this
- doesn't actually decode the docFreq, totalTermFreq,
- postings details (frq/prx offset, etc.) metadata;
- it just loads them as byte[] blobs which are then
- decoded on-demand if the metadata is ever requested
- for any term in this block. This enables terms-only
- intensive consumes (eg certain MTQs, respelling) to
- not pay the price of decoding metadata they won't
- use. */
- void loadBlock() throws IOException {
-
- // Clone the IndexInput lazily, so that consumers
- // that just pull a TermsEnum to
- // seekExact(TermState) don't pay this cost:
- ste.initIndexInput();
-
- if (nextEnt != -1) {
- // Already loaded
- return;
- }
- // System.out.println("blc=" + blockLoadCount);
-
- ste.in.seek(fp);
- int code = ste.in.readVInt();
- entCount = code >>> 1;
- assert entCount > 0;
- isLastInFloor = (code & 1) != 0;
-
- assert arc == null || (isLastInFloor || isFloor)
- : "fp=" + fp + " arc=" + arc + " isFloor=" + isFloor + " isLastInFloor=" + isLastInFloor;
-
- // TODO: if suffixes were stored in random-access
- // array structure, then we could do binary search
- // instead of linear scan to find target term; eg
- // we could have simple array of offsets
-
- final long startSuffixFP = ste.in.getFilePointer();
- // term suffixes:
- if (version >= Lucene40BlockTreeTermsReader.VERSION_COMPRESSED_SUFFIXES) {
- final long codeL = ste.in.readVLong();
- isLeafBlock = (codeL & 0x04) != 0;
- final int numSuffixBytes = (int) (codeL >>> 3);
- if (suffixBytes.length < numSuffixBytes) {
- suffixBytes = new byte[ArrayUtil.oversize(numSuffixBytes, 1)];
- }
- try {
- compressionAlg = CompressionAlgorithm.byCode((int) codeL & 0x03);
- } catch (IllegalArgumentException e) {
- throw new CorruptIndexException(e.getMessage(), ste.in, e);
- }
- compressionAlg.read(ste.in, suffixBytes, numSuffixBytes);
- suffixesReader.reset(suffixBytes, 0, numSuffixBytes);
-
- int numSuffixLengthBytes = ste.in.readVInt();
- final boolean allEqual = (numSuffixLengthBytes & 0x01) != 0;
- numSuffixLengthBytes >>>= 1;
- if (suffixLengthBytes.length < numSuffixLengthBytes) {
- suffixLengthBytes = new byte[ArrayUtil.oversize(numSuffixLengthBytes, 1)];
- }
- if (allEqual) {
- Arrays.fill(suffixLengthBytes, 0, numSuffixLengthBytes, ste.in.readByte());
- } else {
- ste.in.readBytes(suffixLengthBytes, 0, numSuffixLengthBytes);
- }
- suffixLengthsReader.reset(suffixLengthBytes, 0, numSuffixLengthBytes);
- } else {
- code = ste.in.readVInt();
- isLeafBlock = (code & 1) != 0;
- int numBytes = code >>> 1;
- if (suffixBytes.length < numBytes) {
- suffixBytes = new byte[ArrayUtil.oversize(numBytes, 1)];
- }
- ste.in.readBytes(suffixBytes, 0, numBytes);
- suffixesReader.reset(suffixBytes, 0, numBytes);
- }
- totalSuffixBytes = ste.in.getFilePointer() - startSuffixFP;
-
- /*if (DEBUG) {
- if (arc == null) {
- System.out.println(" loadBlock (next) fp=" + fp + " entCount=" + entCount + " prefixLen=" + prefix + " isLastInFloor=" + isLastInFloor + " leaf?=" + isLeafBlock);
- } else {
- System.out.println(" loadBlock (seek) fp=" + fp + " entCount=" + entCount + " prefixLen=" + prefix + " hasTerms?=" + hasTerms + " isFloor?=" + isFloor + " isLastInFloor=" + isLastInFloor + " leaf?=" + isLeafBlock);
- }
- }*/
-
- // stats
- int numBytes = ste.in.readVInt();
- if (statBytes.length < numBytes) {
- statBytes = new byte[ArrayUtil.oversize(numBytes, 1)];
- }
- ste.in.readBytes(statBytes, 0, numBytes);
- statsReader.reset(statBytes, 0, numBytes);
- statsSingletonRunLength = 0;
- metaDataUpto = 0;
-
- state.termBlockOrd = 0;
- nextEnt = 0;
- lastSubFP = -1;
-
- // TODO: we could skip this if !hasTerms; but
- // that's rare so won't help much
- // metadata
- numBytes = ste.in.readVInt();
- if (bytes.length < numBytes) {
- bytes = new byte[ArrayUtil.oversize(numBytes, 1)];
- }
- ste.in.readBytes(bytes, 0, numBytes);
- bytesReader.reset(bytes, 0, numBytes);
-
- // Sub-blocks of a single floor block are always
- // written one after another -- tail recurse:
- fpEnd = ste.in.getFilePointer();
- // if (DEBUG) {
- // System.out.println(" fpEnd=" + fpEnd);
- // }
- }
-
- void rewind() {
-
- // Force reload:
- fp = fpOrig;
- nextEnt = -1;
- hasTerms = hasTermsOrig;
- if (isFloor) {
- floorDataReader.rewind();
- numFollowFloorBlocks = floorDataReader.readVInt();
- assert numFollowFloorBlocks > 0;
- nextFloorLabel = floorDataReader.readByte() & 0xff;
- }
-
- /*
- //System.out.println("rewind");
- // Keeps the block loaded, but rewinds its state:
- if (nextEnt > 0 || fp != fpOrig) {
- if (DEBUG) {
- System.out.println(" rewind frame ord=" + ord + " fpOrig=" + fpOrig + " fp=" + fp + " hasTerms?=" + hasTerms + " isFloor?=" + isFloor + " nextEnt=" + nextEnt + " prefixLen=" + prefix);
- }
- if (fp != fpOrig) {
- fp = fpOrig;
- nextEnt = -1;
- } else {
- nextEnt = 0;
- }
- hasTerms = hasTermsOrig;
- if (isFloor) {
- floorDataReader.rewind();
- numFollowFloorBlocks = floorDataReader.readVInt();
- nextFloorLabel = floorDataReader.readByte() & 0xff;
- }
- assert suffixBytes != null;
- suffixesReader.rewind();
- assert statBytes != null;
- statsReader.rewind();
- metaDataUpto = 0;
- state.termBlockOrd = 0;
- // TODO: skip this if !hasTerms? Then postings
- // impl wouldn't have to write useless 0 byte
- postingsReader.resetTermsBlock(fieldInfo, state);
- lastSubFP = -1;
- } else if (DEBUG) {
- System.out.println(" skip rewind fp=" + fp + " fpOrig=" + fpOrig + " nextEnt=" + nextEnt + " ord=" + ord);
- }
- */
- }
-
- // Decodes next entry; returns true if it's a sub-block
- public boolean next() throws IOException {
- if (isLeafBlock) {
- nextLeaf();
- return false;
- } else {
- return nextNonLeaf();
- }
- }
-
- public void nextLeaf() {
- // if (DEBUG) System.out.println(" frame.next ord=" + ord + " nextEnt=" + nextEnt +
- // " entCount=" + entCount);
- assert nextEnt != -1 && nextEnt < entCount
- : "nextEnt=" + nextEnt + " entCount=" + entCount + " fp=" + fp;
- nextEnt++;
- suffix = suffixLengthsReader.readVInt();
- startBytePos = suffixesReader.getPosition();
- ste.term.setLength(prefix + suffix);
- ste.term.grow(ste.term.length());
- suffixesReader.readBytes(ste.term.bytes(), prefix, suffix);
- ste.termExists = true;
- }
-
- public boolean nextNonLeaf() throws IOException {
- // if (DEBUG) System.out.println(" stef.next ord=" + ord + " nextEnt=" + nextEnt + " entCount="
- // + entCount + " fp=" + suffixesReader.getPosition());
- while (true) {
- if (nextEnt == entCount) {
- assert arc == null || (isFloor && isLastInFloor == false)
- : "isFloor=" + isFloor + " isLastInFloor=" + isLastInFloor;
- loadNextFloorBlock();
- if (isLeafBlock) {
- nextLeaf();
- return false;
- } else {
- continue;
- }
- }
-
- assert nextEnt != -1 && nextEnt < entCount
- : "nextEnt=" + nextEnt + " entCount=" + entCount + " fp=" + fp;
- nextEnt++;
- final int code = suffixLengthsReader.readVInt();
- suffix = code >>> 1;
- startBytePos = suffixesReader.getPosition();
- ste.term.setLength(prefix + suffix);
- ste.term.grow(ste.term.length());
- suffixesReader.readBytes(ste.term.bytes(), prefix, suffix);
- if ((code & 1) == 0) {
- // A normal term
- ste.termExists = true;
- subCode = 0;
- state.termBlockOrd++;
- return false;
- } else {
- // A sub-block; make sub-FP absolute:
- ste.termExists = false;
- subCode = suffixLengthsReader.readVLong();
- lastSubFP = fp - subCode;
- // if (DEBUG) {
- // System.out.println(" lastSubFP=" + lastSubFP);
- // }
- return true;
- }
- }
- }
-
- // TODO: make this array'd so we can do bin search?
- // likely not worth it? need to measure how many
- // floor blocks we "typically" get
- public void scanToFloorFrame(BytesRef target) {
-
- if (!isFloor || target.length <= prefix) {
- // if (DEBUG) {
- // System.out.println(" scanToFloorFrame skip: isFloor=" + isFloor + " target.length=" +
- // target.length + " vs prefix=" + prefix);
- // }
- return;
- }
-
- final int targetLabel = target.bytes[target.offset + prefix] & 0xFF;
-
- // if (DEBUG) {
- // System.out.println(" scanToFloorFrame fpOrig=" + fpOrig + " targetLabel=" +
- // toHex(targetLabel) + " vs nextFloorLabel=" + toHex(nextFloorLabel) + " numFollowFloorBlocks="
- // + numFollowFloorBlocks);
- // }
-
- if (targetLabel < nextFloorLabel) {
- // if (DEBUG) {
- // System.out.println(" already on correct block");
- // }
- return;
- }
-
- assert numFollowFloorBlocks != 0;
-
- long newFP = fpOrig;
- while (true) {
- final long code = floorDataReader.readVLong();
- newFP = fpOrig + (code >>> 1);
- hasTerms = (code & 1) != 0;
- // if (DEBUG) {
- // System.out.println(" label=" + toHex(nextFloorLabel) + " fp=" + newFP +
- // " hasTerms?=" + hasTerms + " numFollowFloor=" + numFollowFloorBlocks);
- // }
-
- isLastInFloor = numFollowFloorBlocks == 1;
- numFollowFloorBlocks--;
-
- if (isLastInFloor) {
- nextFloorLabel = 256;
- // if (DEBUG) {
- // System.out.println(" stop! last block nextFloorLabel=" +
- // toHex(nextFloorLabel));
- // }
- break;
- } else {
- nextFloorLabel = floorDataReader.readByte() & 0xff;
- if (targetLabel < nextFloorLabel) {
- // if (DEBUG) {
- // System.out.println(" stop! nextFloorLabel=" + toHex(nextFloorLabel));
- // }
- break;
- }
- }
- }
-
- if (newFP != fp) {
- // Force re-load of the block:
- // if (DEBUG) {
- // System.out.println(" force switch to fp=" + newFP + " oldFP=" + fp);
- // }
- nextEnt = -1;
- fp = newFP;
- } else {
- // if (DEBUG) {
- // System.out.println(" stay on same fp=" + newFP);
- // }
- }
- }
-
- public void decodeMetaData() throws IOException {
-
- // if (DEBUG) System.out.println("\nBTTR.decodeMetadata seg=" + segment + " mdUpto=" +
- // metaDataUpto + " vs termBlockOrd=" + state.termBlockOrd);
-
- // lazily catch up on metadata decode:
- final int limit = getTermBlockOrd();
- boolean absolute = metaDataUpto == 0;
- assert limit > 0;
-
- // TODO: better API would be "jump straight to term=N"???
- while (metaDataUpto < limit) {
-
- // TODO: we could make "tiers" of metadata, ie,
- // decode docFreq/totalTF but don't decode postings
- // metadata; this way caller could get
- // docFreq/totalTF w/o paying decode cost for
- // postings
-
- // TODO: if docFreq were bulk decoded we could
- // just skipN here:
-
- if (version >= Lucene40BlockTreeTermsReader.VERSION_COMPRESSED_SUFFIXES) {
- if (statsSingletonRunLength > 0) {
- state.docFreq = 1;
- state.totalTermFreq = 1;
- statsSingletonRunLength--;
- } else {
- int token = statsReader.readVInt();
- if ((token & 1) == 1) {
- state.docFreq = 1;
- state.totalTermFreq = 1;
- statsSingletonRunLength = token >>> 1;
- } else {
- state.docFreq = token >>> 1;
- if (ste.fr.fieldInfo.getIndexOptions() == IndexOptions.DOCS) {
- state.totalTermFreq = state.docFreq;
- } else {
- state.totalTermFreq = state.docFreq + statsReader.readVLong();
- }
- }
- }
- } else {
- assert statsSingletonRunLength == 0;
- state.docFreq = statsReader.readVInt();
- // if (DEBUG) System.out.println(" dF=" + state.docFreq);
- if (ste.fr.fieldInfo.getIndexOptions() == IndexOptions.DOCS) {
- state.totalTermFreq = state.docFreq; // all postings have freq=1
- } else {
- state.totalTermFreq = state.docFreq + statsReader.readVLong();
- // if (DEBUG) System.out.println(" totTF=" + state.totalTermFreq);
- }
- }
-
- // metadata
- ste.fr.parent.postingsReader.decodeTerm(bytesReader, ste.fr.fieldInfo, state, absolute);
-
- metaDataUpto++;
- absolute = false;
- }
- state.termBlockOrd = metaDataUpto;
- }
-
- // Used only by assert
- private boolean prefixMatches(BytesRef target) {
- for (int bytePos = 0; bytePos < prefix; bytePos++) {
- if (target.bytes[target.offset + bytePos] != ste.term.byteAt(bytePos)) {
- return false;
- }
- }
-
- return true;
- }
-
- // Scans to sub-block that has this target fp; only
- // called by next(); NOTE: does not set
- // startBytePos/suffix as a side effect
- public void scanToSubBlock(long subFP) {
- assert !isLeafBlock;
- // if (DEBUG) System.out.println(" scanToSubBlock fp=" + fp + " subFP=" + subFP + " entCount="
- // + entCount + " lastSubFP=" + lastSubFP);
- // assert nextEnt == 0;
- if (lastSubFP == subFP) {
- // if (DEBUG) System.out.println(" already positioned");
- return;
- }
- assert subFP < fp : "fp=" + fp + " subFP=" + subFP;
- final long targetSubCode = fp - subFP;
- // if (DEBUG) System.out.println(" targetSubCode=" + targetSubCode);
- while (true) {
- assert nextEnt < entCount;
- nextEnt++;
- final int code = suffixLengthsReader.readVInt();
- suffixesReader.skipBytes(code >>> 1);
- if ((code & 1) != 0) {
- final long subCode = suffixLengthsReader.readVLong();
- if (targetSubCode == subCode) {
- // if (DEBUG) System.out.println(" match!");
- lastSubFP = subFP;
- return;
- }
- } else {
- state.termBlockOrd++;
- }
- }
- }
-
- // NOTE: sets startBytePos/suffix as a side effect
- public SeekStatus scanToTerm(BytesRef target, boolean exactOnly) throws IOException {
- return isLeafBlock ? scanToTermLeaf(target, exactOnly) : scanToTermNonLeaf(target, exactOnly);
- }
-
- private int startBytePos;
- private int suffix;
- private long subCode;
- CompressionAlgorithm compressionAlg = CompressionAlgorithm.NO_COMPRESSION;
-
- // Target's prefix matches this block's prefix; we
- // scan the entries check if the suffix matches.
- public SeekStatus scanToTermLeaf(BytesRef target, boolean exactOnly) throws IOException {
-
- // if (DEBUG) System.out.println(" scanToTermLeaf: block fp=" + fp + " prefix=" + prefix +
- // " nextEnt=" + nextEnt + " (of " + entCount + ") target=" +
- // ToStringUtils.bytesRefToString(target) +
- // " term=" + ToStringUtils.bytesRefToString(term));
-
- assert nextEnt != -1;
-
- ste.termExists = true;
- subCode = 0;
-
- if (nextEnt == entCount) {
- if (exactOnly) {
- fillTerm();
- }
- return SeekStatus.END;
- }
-
- assert prefixMatches(target);
-
- // TODO: binary search when all terms have the same length, which is common for ID fields,
- // which are also the most sensitive to lookup performance?
- // Loop over each entry (term or sub-block) in this block:
- do {
- nextEnt++;
-
- suffix = suffixLengthsReader.readVInt();
-
- // if (DEBUG) {
- // BytesRef suffixBytesRef = new BytesRef();
- // suffixBytesRef.bytes = suffixBytes;
- // suffixBytesRef.offset = suffixesReader.getPosition();
- // suffixBytesRef.length = suffix;
- // System.out.println(" cycle: term " + (nextEnt-1) + " (of " + entCount + ") suffix="
- // + ToStringUtils.bytesRefToString(suffixBytesRef));
- // }
-
- startBytePos = suffixesReader.getPosition();
- suffixesReader.skipBytes(suffix);
-
- // Loop over bytes in the suffix, comparing to the target
- final int cmp =
- Arrays.compareUnsigned(
- suffixBytes,
- startBytePos,
- startBytePos + suffix,
- target.bytes,
- target.offset + prefix,
- target.offset + target.length);
-
- if (cmp < 0) {
- // Current entry is still before the target;
- // keep scanning
- } else if (cmp > 0) {
- // Done! Current entry is after target --
- // return NOT_FOUND:
- fillTerm();
-
- // if (DEBUG) System.out.println(" not found");
- return SeekStatus.NOT_FOUND;
- } else {
- // Exact match!
-
- // This cannot be a sub-block because we
- // would have followed the index to this
- // sub-block from the start:
-
- assert ste.termExists;
- fillTerm();
- // if (DEBUG) System.out.println(" found!");
- return SeekStatus.FOUND;
- }
- } while (nextEnt < entCount);
-
- // It is possible (and OK) that terms index pointed us
- // at this block, but, we scanned the entire block and
- // did not find the term to position to. This happens
- // when the target is after the last term in the block
- // (but, before the next term in the index). EG
- // target could be foozzz, and terms index pointed us
- // to the foo* block, but the last term in this block
- // was fooz (and, eg, first term in the next block will
- // bee fop).
- // if (DEBUG) System.out.println(" block end");
- if (exactOnly) {
- fillTerm();
- }
-
- // TODO: not consistent that in the
- // not-exact case we don't next() into the next
- // frame here
- return SeekStatus.END;
- }
-
- // Target's prefix matches this block's prefix; we
- // scan the entries check if the suffix matches.
- public SeekStatus scanToTermNonLeaf(BytesRef target, boolean exactOnly) throws IOException {
-
- // if (DEBUG) System.out.println(" scanToTermNonLeaf: block fp=" + fp + " prefix=" + prefix +
- // " nextEnt=" + nextEnt + " (of " + entCount + ") target=" +
- // ToStringUtils.bytesRefToString(target) +
- // " term=" + ToStringUtils.bytesRefToString(term));
-
- assert nextEnt != -1;
-
- if (nextEnt == entCount) {
- if (exactOnly) {
- fillTerm();
- ste.termExists = subCode == 0;
- }
- return SeekStatus.END;
- }
-
- assert prefixMatches(target);
-
- // Loop over each entry (term or sub-block) in this block:
- while (nextEnt < entCount) {
-
- nextEnt++;
-
- final int code = suffixLengthsReader.readVInt();
- suffix = code >>> 1;
-
- // if (DEBUG) {
- // BytesRef suffixBytesRef = new BytesRef();
- // suffixBytesRef.bytes = suffixBytes;
- // suffixBytesRef.offset = suffixesReader.getPosition();
- // suffixBytesRef.length = suffix;
- // System.out.println(" cycle: " + ((code&1)==1 ? "sub-block" : "term") + " " +
- // (nextEnt-1) + " (of " + entCount + ") suffix=" +
- // ToStringUtils.bytesRefToString(suffixBytesRef));
- // }
-
- final int termLen = prefix + suffix;
- startBytePos = suffixesReader.getPosition();
- suffixesReader.skipBytes(suffix);
- ste.termExists = (code & 1) == 0;
- if (ste.termExists) {
- state.termBlockOrd++;
- subCode = 0;
- } else {
- subCode = suffixLengthsReader.readVLong();
- lastSubFP = fp - subCode;
- }
-
- final int cmp =
- Arrays.compareUnsigned(
- suffixBytes,
- startBytePos,
- startBytePos + suffix,
- target.bytes,
- target.offset + prefix,
- target.offset + target.length);
-
- if (cmp < 0) {
- // Current entry is still before the target;
- // keep scanning
- } else if (cmp > 0) {
- // Done! Current entry is after target --
- // return NOT_FOUND:
- fillTerm();
-
- // if (DEBUG) System.out.println(" maybe done exactOnly=" + exactOnly +
- // " ste.termExists=" + ste.termExists);
-
- if (!exactOnly && !ste.termExists) {
- // System.out.println(" now pushFrame");
- // TODO this
- // We are on a sub-block, and caller wants
- // us to position to the next term after
- // the target, so we must recurse into the
- // sub-frame(s):
- ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, termLen);
- ste.currentFrame.loadBlock();
- while (ste.currentFrame.next()) {
- ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, ste.term.length());
- ste.currentFrame.loadBlock();
- }
- }
-
- // if (DEBUG) System.out.println(" not found");
- return SeekStatus.NOT_FOUND;
- } else {
- // Exact match!
-
- // This cannot be a sub-block because we
- // would have followed the index to this
- // sub-block from the start:
-
- assert ste.termExists;
- fillTerm();
- // if (DEBUG) System.out.println(" found!");
- return SeekStatus.FOUND;
- }
- }
-
- // It is possible (and OK) that terms index pointed us
- // at this block, but, we scanned the entire block and
- // did not find the term to position to. This happens
- // when the target is after the last term in the block
- // (but, before the next term in the index). EG
- // target could be foozzz, and terms index pointed us
- // to the foo* block, but the last term in this block
- // was fooz (and, eg, first term in the next block will
- // bee fop).
- // if (DEBUG) System.out.println(" block end");
- if (exactOnly) {
- fillTerm();
- }
-
- // TODO: not consistent that in the
- // not-exact case we don't next() into the next
- // frame here
- return SeekStatus.END;
- }
-
- private void fillTerm() {
- final int termLength = prefix + suffix;
- ste.term.setLength(termLength);
- ste.term.grow(termLength);
- System.arraycopy(suffixBytes, startBytePos, ste.term.bytes(), prefix, suffix);
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/Stats.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/Stats.java
deleted file mode 100644
index 812a959e714..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/Stats.java
+++ /dev/null
@@ -1,276 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene40.blocktree;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.io.ByteArrayOutputStream;
-import java.io.PrintStream;
-import java.util.Locale;
-import org.apache.lucene.codecs.PostingsReaderBase;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-
-/**
- * BlockTree statistics for a single field returned by {@link FieldReader#getStats()}.
- *
- * @lucene.internal
- */
-public class Stats {
- /** Byte size of the index. */
- public long indexNumBytes;
-
- /** Total number of terms in the field. */
- public long totalTermCount;
-
- /** Total number of bytes (sum of term lengths) across all terms in the field. */
- public long totalTermBytes;
-
- /** The number of normal (non-floor) blocks in the terms file. */
- public int nonFloorBlockCount;
-
- /**
- * The number of floor blocks (meta-blocks larger than the allowed {@code maxItemsPerBlock}) in
- * the terms file.
- */
- public int floorBlockCount;
-
- /** The number of sub-blocks within the floor blocks. */
- public int floorSubBlockCount;
-
- /** The number of "internal" blocks (that have both terms and sub-blocks). */
- public int mixedBlockCount;
-
- /** The number of "leaf" blocks (blocks that have only terms). */
- public int termsOnlyBlockCount;
-
- /** The number of "internal" blocks that do not contain terms (have only sub-blocks). */
- public int subBlocksOnlyBlockCount;
-
- /** Total number of blocks. */
- public int totalBlockCount;
-
- /** Number of blocks at each prefix depth. */
- public int[] blockCountByPrefixLen = new int[10];
-
- private int startBlockCount;
- private int endBlockCount;
-
- /** Total number of bytes used to store term suffixes. */
- public long totalBlockSuffixBytes;
-
- /**
- * Number of times each compression method has been used. 0 = uncompressed 1 = lowercase_ascii 2 =
- * LZ4
- */
- public final long[] compressionAlgorithms = new long[3];
-
- /** Total number of suffix bytes before compression. */
- public long totalUncompressedBlockSuffixBytes;
-
- /**
- * Total number of bytes used to store term stats (not including what the {@link
- * PostingsReaderBase} stores).
- */
- public long totalBlockStatsBytes;
-
- /**
- * Total bytes stored by the {@link PostingsReaderBase}, plus the other few vInts stored in the
- * frame.
- */
- public long totalBlockOtherBytes;
-
- /** Segment name. */
- public final String segment;
-
- /** Field name. */
- public final String field;
-
- Stats(String segment, String field) {
- this.segment = segment;
- this.field = field;
- }
-
- void startBlock(SegmentTermsEnumFrame frame, boolean isFloor) {
- totalBlockCount++;
- if (isFloor) {
- if (frame.fp == frame.fpOrig) {
- floorBlockCount++;
- }
- floorSubBlockCount++;
- } else {
- nonFloorBlockCount++;
- }
-
- if (blockCountByPrefixLen.length <= frame.prefix) {
- blockCountByPrefixLen = ArrayUtil.grow(blockCountByPrefixLen, 1 + frame.prefix);
- }
- blockCountByPrefixLen[frame.prefix]++;
- startBlockCount++;
- totalBlockSuffixBytes += frame.totalSuffixBytes;
- totalUncompressedBlockSuffixBytes += frame.suffixesReader.length();
- if (frame.suffixesReader != frame.suffixLengthsReader) {
- totalUncompressedBlockSuffixBytes += frame.suffixLengthsReader.length();
- }
- totalBlockStatsBytes += frame.statsReader.length();
- compressionAlgorithms[frame.compressionAlg.code]++;
- }
-
- void endBlock(SegmentTermsEnumFrame frame) {
- final int termCount = frame.isLeafBlock ? frame.entCount : frame.state.termBlockOrd;
- final int subBlockCount = frame.entCount - termCount;
- totalTermCount += termCount;
- if (termCount != 0 && subBlockCount != 0) {
- mixedBlockCount++;
- } else if (termCount != 0) {
- termsOnlyBlockCount++;
- } else if (subBlockCount != 0) {
- subBlocksOnlyBlockCount++;
- } else {
- throw new IllegalStateException();
- }
- endBlockCount++;
- final long otherBytes =
- frame.fpEnd - frame.fp - frame.totalSuffixBytes - frame.statsReader.length();
- assert otherBytes > 0
- : "otherBytes=" + otherBytes + " frame.fp=" + frame.fp + " frame.fpEnd=" + frame.fpEnd;
- totalBlockOtherBytes += otherBytes;
- }
-
- void term(BytesRef term) {
- totalTermBytes += term.length;
- }
-
- void finish() {
- assert startBlockCount == endBlockCount
- : "startBlockCount=" + startBlockCount + " endBlockCount=" + endBlockCount;
- assert totalBlockCount == floorSubBlockCount + nonFloorBlockCount
- : "floorSubBlockCount="
- + floorSubBlockCount
- + " nonFloorBlockCount="
- + nonFloorBlockCount
- + " totalBlockCount="
- + totalBlockCount;
- assert totalBlockCount == mixedBlockCount + termsOnlyBlockCount + subBlocksOnlyBlockCount
- : "totalBlockCount="
- + totalBlockCount
- + " mixedBlockCount="
- + mixedBlockCount
- + " subBlocksOnlyBlockCount="
- + subBlocksOnlyBlockCount
- + " termsOnlyBlockCount="
- + termsOnlyBlockCount;
- }
-
- @Override
- public String toString() {
- final ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
- PrintStream out = new PrintStream(bos, false, UTF_8);
-
- out.println(" index FST:");
- out.println(" " + indexNumBytes + " bytes");
- out.println(" terms:");
- out.println(" " + totalTermCount + " terms");
- out.println(
- " "
- + totalTermBytes
- + " bytes"
- + (totalTermCount != 0
- ? " ("
- + String.format(Locale.ROOT, "%.1f", ((double) totalTermBytes) / totalTermCount)
- + " bytes/term)"
- : ""));
- out.println(" blocks:");
- out.println(" " + totalBlockCount + " blocks");
- out.println(" " + termsOnlyBlockCount + " terms-only blocks");
- out.println(" " + subBlocksOnlyBlockCount + " sub-block-only blocks");
- out.println(" " + mixedBlockCount + " mixed blocks");
- out.println(" " + floorBlockCount + " floor blocks");
- out.println(" " + (totalBlockCount - floorSubBlockCount) + " non-floor blocks");
- out.println(" " + floorSubBlockCount + " floor sub-blocks");
- out.println(
- " "
- + totalUncompressedBlockSuffixBytes
- + " term suffix bytes before compression"
- + (totalBlockCount != 0
- ? " ("
- + String.format(
- Locale.ROOT, "%.1f", ((double) totalBlockSuffixBytes) / totalBlockCount)
- + " suffix-bytes/block)"
- : ""));
- StringBuilder compressionCounts = new StringBuilder();
- for (int code = 0; code < compressionAlgorithms.length; ++code) {
- if (compressionAlgorithms[code] == 0) {
- continue;
- }
- if (compressionCounts.length() > 0) {
- compressionCounts.append(", ");
- }
- compressionCounts.append(CompressionAlgorithm.byCode(code));
- compressionCounts.append(": ");
- compressionCounts.append(compressionAlgorithms[code]);
- }
- out.println(
- " "
- + totalBlockSuffixBytes
- + " compressed term suffix bytes"
- + (totalBlockCount != 0
- ? " ("
- + String.format(
- Locale.ROOT,
- "%.2f",
- ((double) totalBlockSuffixBytes) / totalUncompressedBlockSuffixBytes)
- + " compression ratio - compression count by algorithm: "
- + compressionCounts
- : "")
- + ")");
- out.println(
- " "
- + totalBlockStatsBytes
- + " term stats bytes "
- + (totalBlockCount != 0
- ? " ("
- + String.format(
- Locale.ROOT, "%.1f", ((double) totalBlockStatsBytes) / totalBlockCount)
- + " stats-bytes/block)"
- : ""));
- out.println(
- " "
- + totalBlockOtherBytes
- + " other bytes"
- + (totalBlockCount != 0
- ? " ("
- + String.format(
- Locale.ROOT, "%.1f", ((double) totalBlockOtherBytes) / totalBlockCount)
- + " other-bytes/block)"
- : ""));
- if (totalBlockCount != 0) {
- out.println(" by prefix length:");
- int total = 0;
- for (int prefix = 0; prefix < blockCountByPrefixLen.length; prefix++) {
- final int blockCount = blockCountByPrefixLen[prefix];
- total += blockCount;
- if (blockCount != 0) {
- out.println(" " + String.format(Locale.ROOT, "%2d", prefix) + ": " + blockCount);
- }
- }
- assert totalBlockCount == total;
- }
-
- return bos.toString(UTF_8);
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/package-info.java
deleted file mode 100644
index a34b7aa345d..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/package-info.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * BlockTree terms dictionary.
- *
- * <p>This terms dictionary organizes all terms into blocks according to shared prefix, such that
- * each block has enough terms, and then stores the prefix trie in memory as an FST as the index
- * structure. It allows you to plug in your own {@link org.apache.lucene.codecs.PostingsWriterBase}
- * to implement the postings.
- *
- * <p>See {@code BlockTreeTermsWriter} for the file format.
- */
-package org.apache.lucene.backward_codecs.lucene40.blocktree;
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/ForUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/ForUtil.java
deleted file mode 100644
index b6131049694..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/ForUtil.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.BLOCK_SIZE;
-
-import java.io.IOException;
-import java.util.Arrays;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.packed.PackedInts;
-import org.apache.lucene.util.packed.PackedInts.Decoder;
-import org.apache.lucene.util.packed.PackedInts.FormatAndBits;
-
-/**
- * Encode all values in normal area with fixed bit width, which is determined by the max value in
- * this block.
- */
-final class ForUtil {
-
- /** Special number of bits per value used whenever all values to encode are equal. */
- private static final int ALL_VALUES_EQUAL = 0;
-
- /**
- * Upper limit of the number of bytes that might be required to stored <code>BLOCK_SIZE</code>
- * encoded values.
- */
- static final int MAX_ENCODED_SIZE = BLOCK_SIZE * 4;
-
- /**
- * Upper limit of the number of values that might be decoded in a single call to {@link
- * #readBlock(IndexInput, byte[], int[])}. Although values after <code>BLOCK_SIZE</code> are
- * garbage, it is necessary to allocate value buffers whose size is {@code >= MAX_DATA_SIZE} to
- * avoid {@link ArrayIndexOutOfBoundsException}s.
- */
- static final int MAX_DATA_SIZE;
-
- static {
- int maxDataSize = 0;
- for (int version = PackedInts.VERSION_START; version <= PackedInts.VERSION_CURRENT; version++) {
- for (PackedInts.Format format : PackedInts.Format.values()) {
- for (int bpv = 1; bpv <= 32; ++bpv) {
- if (!format.isSupported(bpv)) {
- continue;
- }
- final PackedInts.Decoder decoder = PackedInts.getDecoder(format, version, bpv);
- final int iterations = computeIterations(decoder);
- maxDataSize = Math.max(maxDataSize, iterations * decoder.byteValueCount());
- }
- }
- }
- MAX_DATA_SIZE = maxDataSize;
- }
-
- /**
- * Compute the number of iterations required to decode <code>BLOCK_SIZE</code> values with the
- * provided {@link Decoder}.
- */
- private static int computeIterations(PackedInts.Decoder decoder) {
- return (int) Math.ceil((float) BLOCK_SIZE / decoder.byteValueCount());
- }
-
- /**
- * Compute the number of bytes required to encode a block of values that require <code>
- * bitsPerValue</code> bits per value with format <code>format</code>.
- */
- private static int encodedSize(
- PackedInts.Format format, int packedIntsVersion, int bitsPerValue) {
- final long byteCount = format.byteCount(packedIntsVersion, BLOCK_SIZE, bitsPerValue);
- assert byteCount >= 0 && byteCount <= Integer.MAX_VALUE : byteCount;
- return (int) byteCount;
- }
-
- private final int[] encodedSizes;
- private final PackedInts.Encoder[] encoders;
- private final PackedInts.Decoder[] decoders;
- private final int[] iterations;
-
- /** Create a new {@link ForUtil} instance and save state into <code>out</code>. */
- ForUtil(float acceptableOverheadRatio, DataOutput out) throws IOException {
- out.writeVInt(PackedInts.VERSION_CURRENT);
- encodedSizes = new int[33];
- encoders = new PackedInts.Encoder[33];
- decoders = new PackedInts.Decoder[33];
- iterations = new int[33];
-
- for (int bpv = 1; bpv <= 32; ++bpv) {
- final FormatAndBits formatAndBits =
- PackedInts.fastestFormatAndBits(BLOCK_SIZE, bpv, acceptableOverheadRatio);
- assert formatAndBits.format().isSupported(formatAndBits.bitsPerValue());
- assert formatAndBits.bitsPerValue() <= 32;
- encodedSizes[bpv] =
- encodedSize(
- formatAndBits.format(), PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue());
- encoders[bpv] =
- PackedInts.getEncoder(
- formatAndBits.format(), PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue());
- decoders[bpv] =
- PackedInts.getDecoder(
- formatAndBits.format(), PackedInts.VERSION_CURRENT, formatAndBits.bitsPerValue());
- iterations[bpv] = computeIterations(decoders[bpv]);
-
- out.writeVInt(formatAndBits.format().getId() << 5 | (formatAndBits.bitsPerValue() - 1));
- }
- }
-
- /** Restore a {@link ForUtil} from a {@link DataInput}. */
- ForUtil(DataInput in) throws IOException {
- int packedIntsVersion = in.readVInt();
- PackedInts.checkVersion(packedIntsVersion);
- encodedSizes = new int[33];
- encoders = new PackedInts.Encoder[33];
- decoders = new PackedInts.Decoder[33];
- iterations = new int[33];
-
- for (int bpv = 1; bpv <= 32; ++bpv) {
- final int code = in.readVInt();
- final int formatId = code >>> 5;
- final int bitsPerValue = (code & 31) + 1;
-
- final PackedInts.Format format = PackedInts.Format.byId(formatId);
- assert format.isSupported(bitsPerValue);
- encodedSizes[bpv] = encodedSize(format, packedIntsVersion, bitsPerValue);
- encoders[bpv] = PackedInts.getEncoder(format, packedIntsVersion, bitsPerValue);
- decoders[bpv] = PackedInts.getDecoder(format, packedIntsVersion, bitsPerValue);
- iterations[bpv] = computeIterations(decoders[bpv]);
- }
- }
-
- /**
- * Write a block of data (<code>For</code> format).
- *
- * @param data the data to write
- * @param encoded a buffer to use to encode data
- * @param out the destination output
- * @throws IOException If there is a low-level I/O error
- */
- void writeBlock(int[] data, byte[] encoded, IndexOutput out) throws IOException {
- if (isAllEqual(data)) {
- out.writeByte((byte) ALL_VALUES_EQUAL);
- out.writeVInt(data[0]);
- return;
- }
-
- final int numBits = bitsRequired(data);
- assert numBits > 0 && numBits <= 32 : numBits;
- final PackedInts.Encoder encoder = encoders[numBits];
- final int iters = iterations[numBits];
- assert iters * encoder.byteValueCount() >= BLOCK_SIZE;
- final int encodedSize = encodedSizes[numBits];
- assert iters * encoder.byteBlockCount() >= encodedSize;
-
- out.writeByte((byte) numBits);
-
- encoder.encode(data, 0, encoded, 0, iters);
- out.writeBytes(encoded, encodedSize);
- }
-
- /**
- * Read the next block of data (<code>For</code> format).
- *
- * @param in the input to use to read data
- * @param encoded a buffer that can be used to store encoded data
- * @param decoded where to write decoded data
- * @throws IOException If there is a low-level I/O error
- */
- void readBlock(IndexInput in, byte[] encoded, int[] decoded) throws IOException {
- final int numBits = in.readByte();
- assert numBits <= 32 : numBits;
-
- if (numBits == ALL_VALUES_EQUAL) {
- final int value = in.readVInt();
- Arrays.fill(decoded, 0, BLOCK_SIZE, value);
- return;
- }
-
- final int encodedSize = encodedSizes[numBits];
- in.readBytes(encoded, 0, encodedSize);
-
- final PackedInts.Decoder decoder = decoders[numBits];
- final int iters = iterations[numBits];
- assert iters * decoder.byteValueCount() >= BLOCK_SIZE;
-
- decoder.decode(encoded, 0, decoded, 0, iters);
- }
-
- /**
- * Skip the next block of data.
- *
- * @param in the input where to read data
- * @throws IOException If there is a low-level I/O error
- */
- void skipBlock(IndexInput in) throws IOException {
- final int numBits = in.readByte();
- if (numBits == ALL_VALUES_EQUAL) {
- in.readVInt();
- return;
- }
- assert numBits > 0 && numBits <= 32 : numBits;
- final int encodedSize = encodedSizes[numBits];
- in.seek(in.getFilePointer() + encodedSize);
- }
-
- private static boolean isAllEqual(final int[] data) {
- final int v = data[0];
- for (int i = 1; i < BLOCK_SIZE; ++i) {
- if (data[i] != v) {
- return false;
- }
- }
- return true;
- }
-
- /** Compute the number of bits required to serialize any of the longs in <code>data</code>. */
- private static int bitsRequired(final int[] data) {
- long or = 0;
- for (int i = 0; i < BLOCK_SIZE; ++i) {
- assert data[i] >= 0;
- or |= data[i];
- }
- return PackedInts.bitsRequired(or);
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50CompoundFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50CompoundFormat.java
deleted file mode 100644
index ca697b2948b..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50CompoundFormat.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import java.io.IOException;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.CompoundDirectory;
-import org.apache.lucene.codecs.CompoundFormat;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-
-/**
- * Lucene 5.0 compound file format
- *
- * <p>Files:
- *
- * <ul>
- * <li><code>.cfs</code>: An optional "virtual" file consisting of all the other index files for
- * systems that frequently run out of file handles.
- * <li><code>.cfe</code>: The "virtual" compound file's entry table holding all entries in the
- * corresponding .cfs file.
- * </ul>
- *
- * <p>Description:
- *
- * <ul>
- * <li>Compound (.cfs) --&gt; Header, FileData <sup>FileCount</sup>, Footer
- * <li>Compound Entry Table (.cfe) --&gt; Header, FileCount, &lt;FileName, DataOffset,
- * DataLength&gt; <sup>FileCount</sup>
- * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
- * <li>FileCount --&gt; {@link DataOutput#writeVInt VInt}
- * <li>DataOffset,DataLength,Checksum --&gt; {@link DataOutput#writeLong UInt64}
- * <li>FileName --&gt; {@link DataOutput#writeString String}
- * <li>FileData --&gt; raw file data
- * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
- * </ul>
- *
- * <p>Notes:
- *
- * <ul>
- * <li>FileCount indicates how many files are contained in this compound file. The entry table
- * that follows has that many entries.
- * <li>Each directory entry contains a long pointer to the start of this file's data section, the
- * files length, and a String with that file's name.
- * </ul>
- */
-public final class Lucene50CompoundFormat extends CompoundFormat {
-
- /** Extension of compound file */
- static final String DATA_EXTENSION = "cfs";
-
- /** Extension of compound file entries */
- static final String ENTRIES_EXTENSION = "cfe";
-
- static final String DATA_CODEC = "Lucene50CompoundData";
- static final String ENTRY_CODEC = "Lucene50CompoundEntries";
- static final int VERSION_START = 0;
- static final int VERSION_CURRENT = VERSION_START;
-
- /** Sole constructor. */
- public Lucene50CompoundFormat() {}
-
- @Override
- public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si) throws IOException {
- return new Lucene50CompoundReader(dir, si);
- }
-
- @Override
- public void write(Directory dir, SegmentInfo si, IOContext context) throws IOException {
- throw new UnsupportedOperationException("Old formats can't be used for writing");
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50CompoundReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50CompoundReader.java
deleted file mode 100644
index 819d45a5328..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50CompoundReader.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.CompoundDirectory;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.IOUtils;
-
-/**
- * Class for accessing a compound stream. This class implements a directory, but is limited to only
- * read operations. Directory methods that would normally modify data throw an exception.
- *
- * @lucene.experimental
- */
-final class Lucene50CompoundReader extends CompoundDirectory {
-
- /** Offset/Length for a slice inside of a compound file */
- public static final class FileEntry {
- long offset;
- long length;
- }
-
- private final Directory directory;
- private final String segmentName;
- private final Map<String, FileEntry> entries;
- private final IndexInput handle;
- private int version;
-
- /** Create a new CompoundFileDirectory. */
- // TODO: we should just pre-strip "entries" and append segment name up-front like simpletext?
- // this need not be a "general purpose" directory anymore (it only writes index files)
- public Lucene50CompoundReader(Directory directory, SegmentInfo si) throws IOException {
- this.directory = directory;
- this.segmentName = si.name;
- String dataFileName =
- IndexFileNames.segmentFileName(segmentName, "", Lucene50CompoundFormat.DATA_EXTENSION);
- String entriesFileName =
- IndexFileNames.segmentFileName(segmentName, "", Lucene50CompoundFormat.ENTRIES_EXTENSION);
- this.entries = readEntries(si.getId(), directory, entriesFileName);
- boolean success = false;
-
- long expectedLength = CodecUtil.indexHeaderLength(Lucene50CompoundFormat.DATA_CODEC, "");
- for (Map.Entry<String, FileEntry> ent : entries.entrySet()) {
- expectedLength += ent.getValue().length;
- }
- expectedLength += CodecUtil.footerLength();
-
- handle = directory.openInput(dataFileName, IOContext.DEFAULT);
- // DirectoryUtil.openInput(directory, dataFileName, context);
- try {
- CodecUtil.checkIndexHeader(
- handle, Lucene50CompoundFormat.DATA_CODEC, version, version, si.getId(), "");
-
- // NOTE: data file is too costly to verify checksum against all the bytes on open,
- // but for now we at least verify proper structure of the checksum footer: which looks
- // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
- // such as file truncation.
- CodecUtil.retrieveChecksum(handle);
-
- // We also validate length, because e.g. if you strip 16 bytes off the .cfs we otherwise
- // would not detect it:
- if (handle.length() != expectedLength) {
- throw new CorruptIndexException(
- "length should be " + expectedLength + " bytes, but is " + handle.length() + " instead",
- handle);
- }
-
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(handle);
- }
- }
- }
-
- /** Helper method that reads CFS entries from an input stream */
- private Map<String, FileEntry> readEntries(
- byte[] segmentID, Directory dir, String entriesFileName) throws IOException {
- Map<String, FileEntry> mapping = null;
- try (ChecksumIndexInput entriesStream =
- EndiannessReverserUtil.openChecksumInput(dir, entriesFileName, IOContext.READONCE)) {
- Throwable priorE = null;
- try {
- version =
- CodecUtil.checkIndexHeader(
- entriesStream,
- Lucene50CompoundFormat.ENTRY_CODEC,
- Lucene50CompoundFormat.VERSION_START,
- Lucene50CompoundFormat.VERSION_CURRENT,
- segmentID,
- "");
-
- mapping = readMapping(entriesStream);
-
- } catch (Throwable exception) {
- priorE = exception;
- } finally {
- CodecUtil.checkFooter(entriesStream, priorE);
- }
- }
- return Collections.unmodifiableMap(mapping);
- }
-
- private Map<String, FileEntry> readMapping(IndexInput entriesStream) throws IOException {
- final int numEntries = entriesStream.readVInt();
- Map<String, FileEntry> mapping = HashMap.newHashMap(numEntries);
- for (int i = 0; i < numEntries; i++) {
- final FileEntry fileEntry = new FileEntry();
- final String id = entriesStream.readString();
- FileEntry previous = mapping.put(id, fileEntry);
- if (previous != null) {
- throw new CorruptIndexException("Duplicate cfs entry id=" + id + " in CFS ", entriesStream);
- }
- fileEntry.offset = entriesStream.readLong();
- fileEntry.length = entriesStream.readLong();
- }
- return mapping;
- }
-
- @Override
- public void close() throws IOException {
- IOUtils.close(handle);
- }
-
- @Override
- public IndexInput openInput(String name, IOContext context) throws IOException {
- ensureOpen();
- final String id = IndexFileNames.stripSegmentName(name);
- final FileEntry entry = entries.get(id);
- if (entry == null) {
- String datFileName =
- IndexFileNames.segmentFileName(segmentName, "", Lucene50CompoundFormat.DATA_EXTENSION);
- throw new FileNotFoundException(
- "No sub-file with id "
- + id
- + " found in compound file \""
- + datFileName
- + "\" (fileName="
- + name
- + " files: "
- + entries.keySet()
- + ")");
- }
- return handle.slice(name, entry.offset, entry.length, context);
- }
-
- /** Returns an array of strings, one for each file in the directory. */
- @Override
- public String[] listAll() {
- ensureOpen();
- String[] res = entries.keySet().toArray(new String[entries.size()]);
-
- // Add the segment name
- for (int i = 0; i < res.length; i++) {
- res[i] = segmentName + res[i];
- }
- return res;
- }
-
- /**
- * Returns the length of a file in the directory.
- *
- * @throws IOException if the file does not exist
- */
- @Override
- public long fileLength(String name) throws IOException {
- ensureOpen();
- FileEntry e = entries.get(IndexFileNames.stripSegmentName(name));
- if (e == null) throw new FileNotFoundException(name);
- return e.length;
- }
-
- @Override
- public String toString() {
- return "CompoundFileDirectory(segment=\"" + segmentName + "\" in dir=" + directory + ")";
- }
-
- @Override
- public Set<String> getPendingDeletions() {
- return Collections.emptySet();
- }
-
- @Override
- public void checkIntegrity() throws IOException {
- CodecUtil.checksumEntireFile(handle);
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50LiveDocsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50LiveDocsFormat.java
deleted file mode 100644
index b0187975742..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50LiveDocsFormat.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import java.io.IOException;
-import java.util.Collection;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.LiveDocsFormat;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.SegmentCommitInfo;
-import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.FixedBitSet;
-
-/**
- * Lucene 5.0 live docs format
- *
- * <p>The .liv file is optional, and only exists when a segment contains deletions.
- *
- * <p>Although per-segment, this file is maintained exterior to compound segment files.
- *
- * <p>Deletions (.liv) --&gt; IndexHeader,Generation,Bits
- *
- * <ul>
- * <li>SegmentHeader --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
- * <li>Bits --&gt; &lt;{@link DataOutput#writeLong Int64}&gt; <sup>LongCount</sup>
- * </ul>
- */
-public final class Lucene50LiveDocsFormat extends LiveDocsFormat {
-
- /** extension of live docs */
- private static final String EXTENSION = "liv";
-
- /** codec of live docs */
- private static final String CODEC_NAME = "Lucene50LiveDocs";
-
- /** supported version range */
- private static final int VERSION_START = 0;
-
- private static final int VERSION_CURRENT = VERSION_START;
-
- /** Sole constructor. */
- public Lucene50LiveDocsFormat() {}
-
- @Override
- public Bits readLiveDocs(Directory dir, SegmentCommitInfo info, IOContext context)
- throws IOException {
- long gen = info.getDelGen();
- String name = IndexFileNames.fileNameFromGeneration(info.info.name, EXTENSION, gen);
- final int length = info.info.maxDoc();
- try (ChecksumIndexInput input = EndiannessReverserUtil.openChecksumInput(dir, name, context)) {
- Throwable priorE = null;
- try {
- CodecUtil.checkIndexHeader(
- input,
- CODEC_NAME,
- VERSION_START,
- VERSION_CURRENT,
- info.info.getId(),
- Long.toString(gen, Character.MAX_RADIX));
-
- FixedBitSet fbs = readFixedBitSet(input, length);
-
- if (fbs.length() - fbs.cardinality() != info.getDelCount()) {
- throw new CorruptIndexException(
- "bits.deleted="
- + (fbs.length() - fbs.cardinality())
- + " info.delcount="
- + info.getDelCount(),
- input);
- }
- return fbs.asReadOnlyBits();
- } catch (Throwable exception) {
- priorE = exception;
- } finally {
- CodecUtil.checkFooter(input, priorE);
- }
- }
- throw new AssertionError();
- }
-
- private FixedBitSet readFixedBitSet(IndexInput input, int length) throws IOException {
- long[] data = new long[FixedBitSet.bits2words(length)];
- for (int i = 0; i < data.length; i++) {
- data[i] = input.readLong();
- }
- return new FixedBitSet(data, length);
- }
-
- /**
- * Note: although this format is only used on older versions, we need to keep the write logic in
- * addition to the read logic. When we delete documents that live in an older segment, we write to
- * the live docs for that segment.
- */
- @Override
- public void writeLiveDocs(
- Bits bits, Directory dir, SegmentCommitInfo info, int newDelCount, IOContext context)
- throws IOException {
- long gen = info.getNextDelGen();
- String name = IndexFileNames.fileNameFromGeneration(info.info.name, EXTENSION, gen);
- int delCount;
- try (IndexOutput output = EndiannessReverserUtil.createOutput(dir, name, context)) {
-
- CodecUtil.writeIndexHeader(
- output,
- CODEC_NAME,
- VERSION_CURRENT,
- info.info.getId(),
- Long.toString(gen, Character.MAX_RADIX));
-
- delCount = writeBits(output, bits);
-
- CodecUtil.writeFooter(output);
- }
- if (delCount != info.getDelCount() + newDelCount) {
- throw new CorruptIndexException(
- "bits.deleted="
- + delCount
- + " info.delcount="
- + info.getDelCount()
- + " newdelcount="
- + newDelCount,
- name);
- }
- }
-
- private int writeBits(IndexOutput output, Bits bits) throws IOException {
- int delCount = 0;
- final int longCount = FixedBitSet.bits2words(bits.length());
- for (int i = 0; i < longCount; ++i) {
- long currentBits = 0;
- for (int j = i << 6, end = Math.min(j + 63, bits.length() - 1); j <= end; ++j) {
- if (bits.get(j)) {
- currentBits |= 1L << j; // mod 64
- } else {
- delCount += 1;
- }
- }
- output.writeLong(currentBits);
- }
- return delCount;
- }
-
- @Override
- public void files(SegmentCommitInfo info, Collection<String> files) throws IOException {
- if (info.hasDeletions()) {
- files.add(IndexFileNames.fileNameFromGeneration(info.info.name, EXTENSION, info.getDelGen()));
- }
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50PostingsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50PostingsFormat.java
deleted file mode 100644
index 1d08136cb99..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50PostingsFormat.java
+++ /dev/null
@@ -1,474 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import java.io.IOException;
-import org.apache.lucene.backward_codecs.lucene40.blocktree.Lucene40BlockTreeTermsReader;
-import org.apache.lucene.codecs.BlockTermState;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.FieldsConsumer;
-import org.apache.lucene.codecs.FieldsProducer;
-import org.apache.lucene.codecs.MultiLevelSkipListWriter;
-import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.PostingsReaderBase;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.index.TermState;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.packed.PackedInts;
-
-/**
- * Lucene 5.0 postings format, which encodes postings in packed integer blocks for fast decode.
- *
- * <p>Basic idea:
- *
- * <ul>
- * <li><b>Packed Blocks and VInt Blocks</b>:
- * <p>In packed blocks, integers are encoded with the same bit width ({@link PackedInts packed
- * format}): the block size (i.e. number of integers inside block) is fixed (currently 128).
- * Additionally blocks that are all the same value are encoded in an optimized way.
- * <p>In VInt blocks, integers are encoded as {@link DataOutput#writeVInt VInt}: the block
- * size is variable.
- * <li><b>Block structure</b>:
- * <p>When the postings are long enough, Lucene50PostingsFormat will try to encode most
- * integer data as a packed block.
- * <p>Take a term with 259 documents as an example, the first 256 document ids are encoded as
- * two packed blocks, while the remaining 3 are encoded as one VInt block.
- * <p>Different kinds of data are always encoded separately into different packed blocks, but
- * may possibly be interleaved into the same VInt block.
- * <p>This strategy is applied to pairs: &lt;document number, frequency&gt;, &lt;position,
- * payload length&gt;, &lt;position, offset start, offset length&gt;, and &lt;position,
- * payload length, offsetstart, offset length&gt;.
- * <li><b>Skipdata settings</b>:
- * <p>The structure of skip table is quite similar to previous version of Lucene. Skip
- * interval is the same as block size, and each skip entry points to the beginning of each
- * block. However, for the first block, skip data is omitted.
- * <li><b>Positions, Payloads, and Offsets</b>:
- * <p>A position is an integer indicating where the term occurs within one document. A payload
- * is a blob of metadata associated with current position. An offset is a pair of integers
- * indicating the tokenized start/end offsets for given term in current position: it is
- * essentially a specialized payload.
- * <p>When payloads and offsets are not omitted, numPositions==numPayloads==numOffsets
- * (assuming a null payload contributes one count). As mentioned in block structure, it is
- * possible to encode these three either combined or separately.
- * <p>In all cases, payloads and offsets are stored together. When encoded as a packed block,
- * position data is separated out as .pos, while payloads and offsets are encoded in .pay
- * (payload metadata will also be stored directly in .pay). When encoded as VInt blocks, all
- * these three are stored interleaved into the .pos (so is payload metadata).
- * <p>With this strategy, the majority of payload and offset data will be outside .pos file.
- * So for queries that require only position data, running on a full index with payloads and
- * offsets, this reduces disk pre-fetches.
- * </ul>
- *
- * <p>Files and detailed format:
- *
- * <ul>
- * <li><code>.tim</code>: <a href="#Termdictionary">Term Dictionary</a>
- * <li><code>.tip</code>: <a href="#Termindex">Term Index</a>
- * <li><code>.doc</code>: <a href="#Frequencies">Frequencies and Skip Data</a>
- * <li><code>.pos</code>: <a href="#Positions">Positions</a>
- * <li><code>.pay</code>: <a href="#Payloads">Payloads and Offsets</a>
- * </ul>
- *
- * <a id="Termdictionary"></a>
- *
- * <dl>
- * <dd><b>Term Dictionary</b>
- * <p>The .tim file contains the list of terms in each field along with per-term statistics
- * (such as docfreq) and pointers to the frequencies, positions, payload and skip data in the
- * .doc, .pos, and .pay files. See {@code BlockTreeTermsWriter} for more details on the
- * format.
- * <p>NOTE: The term dictionary can plug into different postings implementations: the postings
- * writer/reader are actually responsible for encoding and decoding the PostingsHeader and
- * TermMetadata sections described here:
- * <ul>
- * <li>PostingsHeader --&gt; Header, PackedBlockSize
- * <li>TermMetadata --&gt; (DocFPDelta|SingletonDocID), PosFPDelta?, PosVIntBlockFPDelta?,
- * PayFPDelta?, SkipFPDelta?
- * <li>Header, --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
- * <li>PackedBlockSize, SingletonDocID --&gt; {@link DataOutput#writeVInt VInt}
- * <li>DocFPDelta, PosFPDelta, PayFPDelta, PosVIntBlockFPDelta, SkipFPDelta --&gt; {@link
- * DataOutput#writeVLong VLong}
- * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
- * </ul>
- * <p>Notes:
- * <ul>
- * <li>Header is a {@link CodecUtil#writeIndexHeader IndexHeader} storing the version
- * information for the postings.
- * <li>PackedBlockSize is the fixed block size for packed blocks. In packed block, bit width
- * is determined by the largest integer. Smaller block size result in smaller variance
- * among width of integers hence smaller indexes. Larger block size result in more
- * efficient bulk i/o hence better acceleration. This value should always be a multiple
- * of 64, currently fixed as 128 as a tradeoff. It is also the skip interval used to
- * accelerate {@link org.apache.lucene.index.PostingsEnum#advance(int)}.
- * <li>DocFPDelta determines the position of this term's TermFreqs within the .doc file. In
- * particular, it is the difference of file offset between this term's data and previous
- * term's data (or zero, for the first term in the block).On disk it is stored as the
- * difference from previous value in sequence.
- * <li>PosFPDelta determines the position of this term's TermPositions within the .pos file.
- * While PayFPDelta determines the position of this term's &lt;TermPayloads,
- * TermOffsets?&gt; within the .pay file. Similar to DocFPDelta, it is the difference
- * between two file positions (or neglected, for fields that omit payloads and offsets).
- * <li>PosVIntBlockFPDelta determines the position of this term's last TermPosition in last
- * pos packed block within the .pos file. It is synonym for PayVIntBlockFPDelta or
- * OffsetVIntBlockFPDelta. This is actually used to indicate whether it is necessary to
- * load following payloads and offsets from .pos instead of .pay. Every time a new block
- * of positions are to be loaded, the PostingsReader will use this value to check
- * whether current block is packed format or VInt. When packed format, payloads and
- * offsets are fetched from .pay, otherwise from .pos. (this value is neglected when
- * total number of positions i.e. totalTermFreq is less or equal to PackedBlockSize).
- * <li>SkipFPDelta determines the position of this term's SkipData within the .doc file. In
- * particular, it is the length of the TermFreq data. SkipDelta is only stored if
- * DocFreq is not smaller than SkipMinimum (i.e. 128 in Lucene50PostingsFormat).
- * <li>SingletonDocID is an optimization when a term only appears in one document. In this
- * case, instead of writing a file pointer to the .doc file (DocFPDelta), and then a
- * VIntBlock at that location, the single document ID is written to the term dictionary.
- * </ul>
- * </dl>
- *
- * <a id="Termindex"></a>
- *
- * <dl>
- * <dd><b>Term Index</b>
- * <p>The .tip file contains an index into the term dictionary, so that it can be accessed
- * randomly. See {@code BlockTreeTermsWriter} for more details on the format.
- * </dl>
- *
- * <a id="Frequencies"></a>
- *
- * <dl>
- * <dd><b>Frequencies and Skip Data</b>
- * <p>The .doc file contains the lists of documents which contain each term, along with the
- * frequency of the term in that document (except when frequencies are omitted: {@link
- * IndexOptions#DOCS}). It also saves skip data to the beginning of each packed or VInt block,
- * when the length of document list is larger than packed block size.
- * <ul>
- * <li>docFile(.doc) --&gt; Header, &lt;TermFreqs, SkipData?&gt;<sup>TermCount</sup>, Footer
- * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
- * <li>TermFreqs --&gt; &lt;PackedBlock&gt; <sup>PackedDocBlockNum</sup>, VIntBlock?
- * <li>PackedBlock --&gt; PackedDocDeltaBlock, PackedFreqBlock?
- * <li>VIntBlock --&gt; &lt;DocDelta[,
- * Freq?]&gt;<sup>DocFreq-PackedBlockSize*PackedDocBlockNum</sup>
- * <li>SkipData --&gt; &lt;&lt;SkipLevelLength, SkipLevel&gt; <sup>NumSkipLevels-1</sup>,
- * SkipLevel&gt;, SkipDatum?
- * <li>SkipLevel --&gt; &lt;SkipDatum&gt; <sup>TrimmedDocFreq/(PackedBlockSize^(Level +
- * 1))</sup>
- * <li>SkipDatum --&gt; DocSkip, DocFPSkip, &lt;PosFPSkip, PosBlockOffset, PayLength?,
- * PayFPSkip?&gt;?, SkipChildLevelPointer?
- * <li>PackedDocDeltaBlock, PackedFreqBlock --&gt; {@link PackedInts PackedInts}
- * <li>DocDelta, Freq, DocSkip, DocFPSkip, PosFPSkip, PosBlockOffset, PayByteUpto, PayFPSkip
- * --&gt; {@link DataOutput#writeVInt VInt}
- * <li>SkipChildLevelPointer --&gt; {@link DataOutput#writeVLong VLong}
- * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
- * </ul>
- * <p>Notes:
- * <ul>
- * <li>PackedDocDeltaBlock is theoretically generated from two steps:
- * <ol>
- * <li>Calculate the difference between each document number and previous one, and get
- * a d-gaps list (for the first document, use absolute value);
- * <li>For those d-gaps from first one to
- * PackedDocBlockNum*PackedBlockSize<sup>th</sup>, separately encode as packed
- * blocks.
- * </ol>
- * If frequencies are not omitted, PackedFreqBlock will be generated without d-gap step.
- * <li>VIntBlock stores remaining d-gaps (along with frequencies when possible) with a
- * format that encodes DocDelta and Freq:
- * <p>DocDelta: if frequencies are indexed, this determines both the document number and
- * the frequency. In particular, DocDelta/2 is the difference between this document
- * number and the previous document number (or zero when this is the first document in a
- * TermFreqs). When DocDelta is odd, the frequency is one. When DocDelta is even, the
- * frequency is read as another VInt. If frequencies are omitted, DocDelta contains the
- * gap (not multiplied by 2) between document numbers and no frequency information is
- * stored.
- * <p>For example, the TermFreqs for a term which occurs once in document seven and
- * three times in document eleven, with frequencies indexed, would be the following
- * sequence of VInts:
- * <p>15, 8, 3
- * <p>If frequencies were omitted ({@link IndexOptions#DOCS}) it would be this sequence
- * of VInts instead:
- * <p>7,4
- * <li>PackedDocBlockNum is the number of packed blocks for current term's docids or
- * frequencies. In particular, PackedDocBlockNum = floor(DocFreq/PackedBlockSize)
- * <li>TrimmedDocFreq = DocFreq % PackedBlockSize == 0 ? DocFreq - 1 : DocFreq. We use this
- * trick since the definition of skip entry is a little different from base interface.
- * In {@link MultiLevelSkipListWriter}, skip data is assumed to be saved for
- * skipInterval<sup>th</sup>, 2*skipInterval<sup>th</sup> ... posting in the list.
- * However, in Lucene50PostingsFormat, the skip data is saved for
- * skipInterval+1<sup>th</sup>, 2*skipInterval+1<sup>th</sup> ... posting
- * (skipInterval==PackedBlockSize in this case). When DocFreq is multiple of
- * PackedBlockSize, MultiLevelSkipListWriter will expect one more skip data than
- * Lucene50SkipWriter.
- * <li>SkipDatum is the metadata of one skip entry. For the first block (no matter packed or
- * VInt), it is omitted.
- * <li>DocSkip records the document number of every PackedBlockSize<sup>th</sup> document
- * number in the postings (i.e. last document number in each packed block). On disk it
- * is stored as the difference from previous value in the sequence.
- * <li>DocFPSkip records the file offsets of each block (excluding )posting at
- * PackedBlockSize+1<sup>th</sup>, 2*PackedBlockSize+1<sup>th</sup> ... , in DocFile.
- * The file offsets are relative to the start of current term's TermFreqs. On disk it is
- * also stored as the difference from previous SkipDatum in the sequence.
- * <li>Since positions and payloads are also block encoded, the skip should skip to related
- * block first, then fetch the values according to in-block offset. PosFPSkip and
- * PayFPSkip record the file offsets of related block in .pos and .pay, respectively.
- * While PosBlockOffset indicates which value to fetch inside the related block
- * (PayBlockOffset is unnecessary since it is always equal to PosBlockOffset). Same as
- * DocFPSkip, the file offsets are relative to the start of current term's TermFreqs,
- * and stored as a difference sequence.
- * <li>PayByteUpto indicates the start offset of the current payload. It is equivalent to
- * the sum of the payload lengths in the current block up to PosBlockOffset
- * </ul>
- * </dl>
- *
- * <a id="Positions"></a>
- *
- * <dl>
- * <dd><b>Positions</b>
- * <p>The .pos file contains the lists of positions that each term occurs at within documents.
- * It also sometimes stores part of payloads and offsets for speedup.
- * <ul>
- * <li>PosFile(.pos) --&gt; Header, &lt;TermPositions&gt; <sup>TermCount</sup>, Footer
- * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
- * <li>TermPositions --&gt; &lt;PackedPosDeltaBlock&gt; <sup>PackedPosBlockNum</sup>,
- * VIntBlock?
- * <li>VIntBlock --&gt; &lt;PositionDelta[, PayloadLength?], PayloadData?, OffsetDelta?,
- * OffsetLength?&gt;<sup>PosVIntCount</sup>
- * <li>PackedPosDeltaBlock --&gt; {@link PackedInts PackedInts}
- * <li>PositionDelta, OffsetDelta, OffsetLength --&gt; {@link DataOutput#writeVInt VInt}
- * <li>PayloadData --&gt; {@link DataOutput#writeByte byte}<sup>PayLength</sup>
- * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
- * </ul>
- * <p>Notes:
- * <ul>
- * <li>TermPositions are order by term (terms are implicit, from the term dictionary), and
- * position values for each term document pair are incremental, and ordered by document
- * number.
- * <li>PackedPosBlockNum is the number of packed blocks for current term's positions,
- * payloads or offsets. In particular, PackedPosBlockNum =
- * floor(totalTermFreq/PackedBlockSize)
- * <li>PosVIntCount is the number of positions encoded as VInt format. In particular,
- * PosVIntCount = totalTermFreq - PackedPosBlockNum*PackedBlockSize
- * <li>The procedure how PackedPosDeltaBlock is generated is the same as PackedDocDeltaBlock
- * in chapter <a href="#Frequencies">Frequencies and Skip Data</a>.
- * <li>PositionDelta is, if payloads are disabled for the term's field, the difference
- * between the position of the current occurrence in the document and the previous
- * occurrence (or zero, if this is the first occurrence in this document). If payloads
- * are enabled for the term's field, then PositionDelta/2 is the difference between the
- * current and the previous position. If payloads are enabled and PositionDelta is odd,
- * then PayloadLength is stored, indicating the length of the payload at the current
- * term position.
- * <li>For example, the TermPositions for a term which occurs as the fourth term in one
- * document, and as the fifth and ninth term in a subsequent document, would be the
- * following sequence of VInts (payloads disabled):
- * <p>4, 5, 4
- * <li>PayloadData is metadata associated with the current term position. If PayloadLength
- * is stored at the current position, then it indicates the length of this payload. If
- * PayloadLength is not stored, then this payload has the same length as the payload at
- * the previous position.
- * <li>OffsetDelta/2 is the difference between this position's startOffset from the previous
- * occurrence (or zero, if this is the first occurrence in this document). If
- * OffsetDelta is odd, then the length (endOffset-startOffset) differs from the previous
- * occurrence and an OffsetLength follows. Offset data is only written for {@link
- * IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}.
- * </ul>
- * </dl>
- *
- * <a id="Payloads"></a>
- *
- * <dl>
- * <dd><b>Payloads and Offsets</b>
- * <p>The .pay file will store payloads and offsets associated with certain term-document
- * positions. Some payloads and offsets will be separated out into .pos file, for performance
- * reasons.
- * <ul>
- * <li>PayFile(.pay): --&gt; Header, &lt;TermPayloads, TermOffsets?&gt;
- * <sup>TermCount</sup>, Footer
- * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
- * <li>TermPayloads --&gt; &lt;PackedPayLengthBlock, SumPayLength, PayData&gt;
- * <sup>PackedPayBlockNum</sup>
- * <li>TermOffsets --&gt; &lt;PackedOffsetStartDeltaBlock, PackedOffsetLengthBlock&gt;
- * <sup>PackedPayBlockNum</sup>
- * <li>PackedPayLengthBlock, PackedOffsetStartDeltaBlock, PackedOffsetLengthBlock --&gt;
- * {@link PackedInts PackedInts}
- * <li>SumPayLength --&gt; {@link DataOutput#writeVInt VInt}
- * <li>PayData --&gt; {@link DataOutput#writeByte byte}<sup>SumPayLength</sup>
- * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
- * </ul>
- * <p>Notes:
- * <ul>
- * <li>The order of TermPayloads/TermOffsets will be the same as TermPositions, note that
- * part of payload/offsets are stored in .pos.
- * <li>The procedure how PackedPayLengthBlock and PackedOffsetLengthBlock are generated is
- * the same as PackedFreqBlock in chapter <a href="#Frequencies">Frequencies and Skip
- * Data</a>. While PackedStartDeltaBlock follows a same procedure as
- * PackedDocDeltaBlock.
- * <li>PackedPayBlockNum is always equal to PackedPosBlockNum, for the same term. It is also
- * synonym for PackedOffsetBlockNum.
- * <li>SumPayLength is the total length of payloads written within one block, should be the
- * sum of PayLengths in one packed block.
- * <li>PayLength in PackedPayLengthBlock is the length of each payload associated with the
- * current position.
- * </ul>
- * </dl>
- *
- * @lucene.experimental
- */
-public class Lucene50PostingsFormat extends PostingsFormat {
-
- /**
- * Filename extension for document number, frequencies, and skip data. See chapter: <a
- * href="#Frequencies">Frequencies and Skip Data</a>
- */
- public static final String DOC_EXTENSION = "doc";
-
- /** Filename extension for positions. See chapter: <a href="#Positions">Positions</a> */
- public static final String POS_EXTENSION = "pos";
-
- /**
- * Filename extension for payloads and offsets. See chapter: <a href="#Payloads">Payloads and
- * Offsets</a>
- */
- public static final String PAY_EXTENSION = "pay";
-
- /**
- * Expert: The maximum number of skip levels. Smaller values result in slightly smaller indexes,
- * but slower skipping in big posting lists.
- */
- static final int MAX_SKIP_LEVELS = 10;
-
- static final String TERMS_CODEC = "Lucene50PostingsWriterTerms";
- static final String DOC_CODEC = "Lucene50PostingsWriterDoc";
- static final String POS_CODEC = "Lucene50PostingsWriterPos";
- static final String PAY_CODEC = "Lucene50PostingsWriterPay";
-
- // Increment version to change it
- static final int VERSION_START = 0;
- static final int VERSION_IMPACT_SKIP_DATA = 1;
- static final int VERSION_CURRENT = VERSION_IMPACT_SKIP_DATA;
-
- /** Fixed packed block size, number of integers encoded in a single packed block. */
- // NOTE: must be multiple of 64 because of PackedInts long-aligned encoding/decoding
- public static final int BLOCK_SIZE = 128;
-
- /** Creates {@code Lucene50PostingsFormat} with default settings. */
- public Lucene50PostingsFormat() {
- super("Lucene50");
- }
-
- @Override
- public String toString() {
- return getName() + "(blocksize=" + BLOCK_SIZE + ")";
- }
-
- @Override
- public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- throw new UnsupportedOperationException("Old formats can't be used for writing");
- }
-
- @Override
- public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new Lucene50PostingsReader(state);
- boolean success = false;
- try {
- FieldsProducer ret = new Lucene40BlockTreeTermsReader(postingsReader, state);
- success = true;
- return ret;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(postingsReader);
- }
- }
- }
-
- /**
- * Holds all state required for {@link Lucene50PostingsReader} to produce a {@link
- * org.apache.lucene.index.PostingsEnum} without re-seeking the terms dict.
- *
- * @lucene.internal
- */
- public static final class IntBlockTermState extends BlockTermState {
- /** file pointer to the start of the doc ids enumeration, in {@link #DOC_EXTENSION} file */
- public long docStartFP;
-
- /** file pointer to the start of the positions enumeration, in {@link #POS_EXTENSION} file */
- public long posStartFP;
-
- /** file pointer to the start of the payloads enumeration, in {@link #PAY_EXTENSION} file */
- public long payStartFP;
-
- /**
- * file offset for the start of the skip list, relative to docStartFP, if there are more than
- * {@link #BLOCK_SIZE} docs; otherwise -1
- */
- public long skipOffset;
-
- /**
- * file offset for the last position in the last block, if there are more than {@link
- * #BLOCK_SIZE} positions; otherwise -1
- */
- public long lastPosBlockOffset;
-
- /**
- * docid when there is a single pulsed posting, otherwise -1. freq is always implicitly
- * totalTermFreq in this case.
- */
- public int singletonDocID;
-
- /** Sole constructor. */
- public IntBlockTermState() {
- skipOffset = -1;
- lastPosBlockOffset = -1;
- singletonDocID = -1;
- }
-
- @Override
- public IntBlockTermState clone() {
- IntBlockTermState other = new IntBlockTermState();
- other.copyFrom(this);
- return other;
- }
-
- @Override
- public void copyFrom(TermState _other) {
- super.copyFrom(_other);
- IntBlockTermState other = (IntBlockTermState) _other;
- docStartFP = other.docStartFP;
- posStartFP = other.posStartFP;
- payStartFP = other.payStartFP;
- lastPosBlockOffset = other.lastPosBlockOffset;
- skipOffset = other.skipOffset;
- singletonDocID = other.singletonDocID;
- }
-
- @Override
- public String toString() {
- return super.toString()
- + " docStartFP="
- + docStartFP
- + " posStartFP="
- + posStartFP
- + " payStartFP="
- + payStartFP
- + " lastPosBlockOffset="
- + lastPosBlockOffset
- + " singletonDocID="
- + singletonDocID;
- }
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50PostingsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50PostingsReader.java
deleted file mode 100644
index fcf700b4a16..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50PostingsReader.java
+++ /dev/null
@@ -1,1866 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import static org.apache.lucene.backward_codecs.lucene50.ForUtil.MAX_DATA_SIZE;
-import static org.apache.lucene.backward_codecs.lucene50.ForUtil.MAX_ENCODED_SIZE;
-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.BLOCK_SIZE;
-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.DOC_CODEC;
-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.MAX_SKIP_LEVELS;
-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.PAY_CODEC;
-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.POS_CODEC;
-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.TERMS_CODEC;
-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.VERSION_CURRENT;
-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.VERSION_START;
-
-import java.io.IOException;
-import java.util.Arrays;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.IntBlockTermState;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.BlockTermState;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.PostingsReaderBase;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.Impacts;
-import org.apache.lucene.index.ImpactsEnum;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.index.SlowImpactsEnum;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
-
-/**
- * Concrete class that reads docId(maybe frq,pos,offset,payloads) list with postings format.
- *
- * @lucene.experimental
- */
-public final class Lucene50PostingsReader extends PostingsReaderBase {
-
- private final IndexInput docIn;
- private final IndexInput posIn;
- private final IndexInput payIn;
-
- final ForUtil forUtil;
- private int version;
-
- /** Sole constructor. */
- public Lucene50PostingsReader(SegmentReadState state) throws IOException {
- boolean success = false;
- IndexInput docIn = null;
- IndexInput posIn = null;
- IndexInput payIn = null;
-
- // NOTE: these data files are too costly to verify checksum against all the bytes on open,
- // but for now we at least verify proper structure of the checksum footer: which looks
- // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
- // such as file truncation.
-
- String docName =
- IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, Lucene50PostingsFormat.DOC_EXTENSION);
- try {
- docIn = EndiannessReverserUtil.openInput(state.directory, docName, state.context);
- version =
- CodecUtil.checkIndexHeader(
- docIn,
- DOC_CODEC,
- VERSION_START,
- VERSION_CURRENT,
- state.segmentInfo.getId(),
- state.segmentSuffix);
- forUtil = new ForUtil(docIn);
- CodecUtil.retrieveChecksum(docIn);
-
- if (state.fieldInfos.hasProx()) {
- String proxName =
- IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, Lucene50PostingsFormat.POS_EXTENSION);
- posIn = EndiannessReverserUtil.openInput(state.directory, proxName, state.context);
- CodecUtil.checkIndexHeader(
- posIn, POS_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);
- CodecUtil.retrieveChecksum(posIn);
-
- if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {
- String payName =
- IndexFileNames.segmentFileName(
- state.segmentInfo.name,
- state.segmentSuffix,
- Lucene50PostingsFormat.PAY_EXTENSION);
- payIn = EndiannessReverserUtil.openInput(state.directory, payName, state.context);
- CodecUtil.checkIndexHeader(
- payIn, PAY_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);
- CodecUtil.retrieveChecksum(payIn);
- }
- }
-
- this.docIn = docIn;
- this.posIn = posIn;
- this.payIn = payIn;
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(docIn, posIn, payIn);
- }
- }
- }
-
- @Override
- public void init(IndexInput termsIn, SegmentReadState state) throws IOException {
- // Make sure we are talking to the matching postings writer
- CodecUtil.checkIndexHeader(
- termsIn,
- TERMS_CODEC,
- VERSION_START,
- VERSION_CURRENT,
- state.segmentInfo.getId(),
- state.segmentSuffix);
- final int indexBlockSize = termsIn.readVInt();
- if (indexBlockSize != BLOCK_SIZE) {
- throw new IllegalStateException(
- "index-time BLOCK_SIZE ("
- + indexBlockSize
- + ") != read-time BLOCK_SIZE ("
- + BLOCK_SIZE
- + ")");
- }
- }
-
- /** Read values that have been written using variable-length encoding instead of bit-packing. */
- static void readVIntBlock(
- IndexInput docIn, int[] docBuffer, int[] freqBuffer, int num, boolean indexHasFreq)
- throws IOException {
- if (indexHasFreq) {
- for (int i = 0; i < num; i++) {
- final int code = docIn.readVInt();
- docBuffer[i] = code >>> 1;
- if ((code & 1) != 0) {
- freqBuffer[i] = 1;
- } else {
- freqBuffer[i] = docIn.readVInt();
- }
- }
- } else {
- for (int i = 0; i < num; i++) {
- docBuffer[i] = docIn.readVInt();
- }
- }
- }
-
- @Override
- public BlockTermState newTermState() {
- return new IntBlockTermState();
- }
-
- @Override
- public void close() throws IOException {
- IOUtils.close(docIn, posIn, payIn);
- }
-
- @Override
- public void decodeTerm(
- DataInput in, FieldInfo fieldInfo, BlockTermState _termState, boolean absolute)
- throws IOException {
- final IntBlockTermState termState = (IntBlockTermState) _termState;
- final boolean fieldHasPositions =
- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- final boolean fieldHasOffsets =
- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
- >= 0;
- final boolean fieldHasPayloads = fieldInfo.hasPayloads();
-
- if (absolute) {
- termState.docStartFP = 0;
- termState.posStartFP = 0;
- termState.payStartFP = 0;
- }
-
- termState.docStartFP += in.readVLong();
- if (fieldHasPositions) {
- termState.posStartFP += in.readVLong();
- if (fieldHasOffsets || fieldHasPayloads) {
- termState.payStartFP += in.readVLong();
- }
- }
- if (termState.docFreq == 1) {
- termState.singletonDocID = in.readVInt();
- } else {
- termState.singletonDocID = -1;
- }
- if (fieldHasPositions) {
- if (termState.totalTermFreq > BLOCK_SIZE) {
- termState.lastPosBlockOffset = in.readVLong();
- } else {
- termState.lastPosBlockOffset = -1;
- }
- }
- if (termState.docFreq > BLOCK_SIZE) {
- termState.skipOffset = in.readVLong();
- } else {
- termState.skipOffset = -1;
- }
- }
-
- @Override
- public PostingsEnum postings(
- FieldInfo fieldInfo, BlockTermState termState, PostingsEnum reuse, int flags)
- throws IOException {
-
- boolean indexHasPositions =
- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
-
- if (indexHasPositions == false
- || PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS) == false) {
- BlockDocsEnum docsEnum;
- if (reuse instanceof BlockDocsEnum) {
- docsEnum = (BlockDocsEnum) reuse;
- if (!docsEnum.canReuse(docIn, fieldInfo)) {
- docsEnum = new BlockDocsEnum(fieldInfo);
- }
- } else {
- docsEnum = new BlockDocsEnum(fieldInfo);
- }
- return docsEnum.reset((IntBlockTermState) termState, flags);
- } else {
- EverythingEnum everythingEnum;
- if (reuse instanceof EverythingEnum) {
- everythingEnum = (EverythingEnum) reuse;
- if (!everythingEnum.canReuse(docIn, fieldInfo)) {
- everythingEnum = new EverythingEnum(fieldInfo);
- }
- } else {
- everythingEnum = new EverythingEnum(fieldInfo);
- }
- return everythingEnum.reset((IntBlockTermState) termState, flags);
- }
- }
-
- @Override
- public ImpactsEnum impacts(FieldInfo fieldInfo, BlockTermState state, int flags)
- throws IOException {
- if (state.docFreq <= BLOCK_SIZE || version < Lucene50PostingsFormat.VERSION_IMPACT_SKIP_DATA) {
- // no skip data
- return new SlowImpactsEnum(postings(fieldInfo, state, null, flags));
- }
-
- final boolean indexHasPositions =
- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- final boolean indexHasOffsets =
- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
- >= 0;
- final boolean indexHasPayloads = fieldInfo.hasPayloads();
-
- if (indexHasPositions
- && PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)
- && (indexHasOffsets == false
- || PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS) == false)
- && (indexHasPayloads == false
- || PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS) == false)) {
- return new BlockImpactsPostingsEnum(fieldInfo, (IntBlockTermState) state);
- }
-
- return new BlockImpactsEverythingEnum(fieldInfo, (IntBlockTermState) state, flags);
- }
-
- final class BlockDocsEnum extends PostingsEnum {
- private final byte[] encoded;
-
- private final int[] docDeltaBuffer = new int[MAX_DATA_SIZE];
- private final int[] freqBuffer = new int[MAX_DATA_SIZE];
-
- private int docBufferUpto;
-
- private Lucene50SkipReader skipper;
- private boolean skipped;
-
- final IndexInput startDocIn;
-
- IndexInput docIn;
- final boolean indexHasFreq;
- final boolean indexHasPos;
- final boolean indexHasOffsets;
- final boolean indexHasPayloads;
-
- private int docFreq; // number of docs in this posting list
- private long totalTermFreq; // sum of freqs in this posting list (or docFreq when omitted)
- private int docUpto; // how many docs we've read
- private int doc; // doc we last read
- private int accum; // accumulator for doc deltas
-
- // Where this term's postings start in the .doc file:
- private long docTermStartFP;
-
- // Where this term's skip data starts (after
- // docTermStartFP) in the .doc file (or -1 if there is
- // no skip data for this term):
- private long skipOffset;
-
- // docID for next skip point, we won't use skipper if
- // target docID is not larger than this
- private int nextSkipDoc;
-
- private boolean needsFreq; // true if the caller actually needs frequencies
- // as we read freqs lazily, isFreqsRead shows if freqs are read for the current block
- // always true when we don't have freqs (indexHasFreq=false) or don't need freqs
- // (needsFreq=false)
- private boolean isFreqsRead;
- private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1
-
- public BlockDocsEnum(FieldInfo fieldInfo) throws IOException {
- this.startDocIn = Lucene50PostingsReader.this.docIn;
- this.docIn = null;
- indexHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
- indexHasPos =
- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- indexHasOffsets =
- fieldInfo
- .getIndexOptions()
- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
- >= 0;
- indexHasPayloads = fieldInfo.hasPayloads();
- encoded = new byte[MAX_ENCODED_SIZE];
- }
-
- public boolean canReuse(IndexInput docIn, FieldInfo fieldInfo) {
- return docIn == startDocIn
- && indexHasFreq
- == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0)
- && indexHasPos
- == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
- >= 0)
- && indexHasPayloads == fieldInfo.hasPayloads();
- }
-
- public PostingsEnum reset(IntBlockTermState termState, int flags) throws IOException {
- docFreq = termState.docFreq;
- totalTermFreq = indexHasFreq ? termState.totalTermFreq : docFreq;
- docTermStartFP = termState.docStartFP;
- skipOffset = termState.skipOffset;
- singletonDocID = termState.singletonDocID;
- if (docFreq > 1) {
- if (docIn == null) {
- // lazy init
- docIn = startDocIn.clone();
- }
- docIn.seek(docTermStartFP);
- }
-
- doc = -1;
- this.needsFreq = PostingsEnum.featureRequested(flags, PostingsEnum.FREQS);
- this.isFreqsRead = true;
- if (indexHasFreq == false || needsFreq == false) {
- Arrays.fill(freqBuffer, 1);
- }
- accum = 0;
- docUpto = 0;
- nextSkipDoc = BLOCK_SIZE - 1; // we won't skip if target is found in first block
- docBufferUpto = BLOCK_SIZE;
- skipped = false;
- return this;
- }
-
- @Override
- public int freq() throws IOException {
- if (isFreqsRead == false) {
- forUtil.readBlock(docIn, encoded, freqBuffer); // read freqs for this block
- isFreqsRead = true;
- }
- return freqBuffer[docBufferUpto - 1];
- }
-
- @Override
- public int nextPosition() throws IOException {
- return -1;
- }
-
- @Override
- public int startOffset() throws IOException {
- return -1;
- }
-
- @Override
- public int endOffset() throws IOException {
- return -1;
- }
-
- @Override
- public BytesRef getPayload() throws IOException {
- return null;
- }
-
- @Override
- public int docID() {
- return doc;
- }
-
- private void refillDocs() throws IOException {
- // Check if we skipped reading the previous block of freqs, and if yes, position docIn after
- // it
- if (isFreqsRead == false) {
- forUtil.skipBlock(docIn);
- isFreqsRead = true;
- }
-
- final int left = docFreq - docUpto;
- assert left > 0;
-
- if (left >= BLOCK_SIZE) {
- forUtil.readBlock(docIn, encoded, docDeltaBuffer);
-
- if (indexHasFreq) {
- if (needsFreq) {
- isFreqsRead = false;
- } else {
- forUtil.skipBlock(docIn); // skip over freqs if we don't need them at all
- }
- }
- } else if (docFreq == 1) {
- docDeltaBuffer[0] = singletonDocID;
- freqBuffer[0] = (int) totalTermFreq;
- } else {
- // Read vInts:
- readVIntBlock(docIn, docDeltaBuffer, freqBuffer, left, indexHasFreq);
- }
- docBufferUpto = 0;
- }
-
- @Override
- public int nextDoc() throws IOException {
- if (docUpto == docFreq) {
- return doc = NO_MORE_DOCS;
- }
- if (docBufferUpto == BLOCK_SIZE) {
- refillDocs(); // we don't need to load freqs for now (will be loaded later if necessary)
- }
-
- accum += docDeltaBuffer[docBufferUpto];
- docUpto++;
-
- doc = accum;
- docBufferUpto++;
- return doc;
- }
-
- @Override
- public int advance(int target) throws IOException {
- // current skip docID < docIDs generated from current buffer <= next skip docID
- // we don't need to skip if target is buffered already
- if (docFreq > BLOCK_SIZE && target > nextSkipDoc) {
-
- if (skipper == null) {
- // Lazy init: first time this enum has ever been used for skipping
- skipper =
- new Lucene50SkipReader(
- version,
- docIn.clone(),
- MAX_SKIP_LEVELS,
- indexHasPos,
- indexHasOffsets,
- indexHasPayloads);
- }
-
- if (!skipped) {
- assert skipOffset != -1;
- // This is the first time this enum has skipped
- // since reset() was called; load the skip data:
- skipper.init(docTermStartFP + skipOffset, docTermStartFP, 0, 0, docFreq);
- skipped = true;
- }
-
- // always plus one to fix the result, since skip position in Lucene50SkipReader
- // is a little different from MultiLevelSkipListReader
- final int newDocUpto = skipper.skipTo(target) + 1;
-
- if (newDocUpto > docUpto) {
- // Skipper moved
- assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;
- docUpto = newDocUpto;
-
- // Force to read next block
- docBufferUpto = BLOCK_SIZE;
- accum = skipper.getDoc(); // actually, this is just lastSkipEntry
- docIn.seek(skipper.getDocPointer()); // now point to the block we want to search
- // even if freqs were not read from the previous block, we will mark them as read,
- // as we don't need to skip the previous block freqs in refillDocs,
- // as we have already positioned docIn where in needs to be.
- isFreqsRead = true;
- }
- // next time we call advance, this is used to
- // foresee whether skipper is necessary.
- nextSkipDoc = skipper.getNextSkipDoc();
- }
- if (docUpto == docFreq) {
- return doc = NO_MORE_DOCS;
- }
- if (docBufferUpto == BLOCK_SIZE) {
- refillDocs();
- }
-
- // Now scan... this is an inlined/pared down version
- // of nextDoc():
- while (true) {
- accum += docDeltaBuffer[docBufferUpto];
- docUpto++;
-
- if (accum >= target) {
- break;
- }
- docBufferUpto++;
- if (docUpto == docFreq) {
- return doc = NO_MORE_DOCS;
- }
- }
-
- docBufferUpto++;
- return doc = accum;
- }
-
- @Override
- public long cost() {
- return docFreq;
- }
- }
-
- // Also handles payloads + offsets
- final class EverythingEnum extends PostingsEnum {
-
- private final byte[] encoded;
-
- private final int[] docDeltaBuffer = new int[MAX_DATA_SIZE];
- private final int[] freqBuffer = new int[MAX_DATA_SIZE];
- private final int[] posDeltaBuffer = new int[MAX_DATA_SIZE];
-
- private final int[] payloadLengthBuffer;
- private final int[] offsetStartDeltaBuffer;
- private final int[] offsetLengthBuffer;
-
- private byte[] payloadBytes;
- private int payloadByteUpto;
- private int payloadLength;
-
- private int lastStartOffset;
- private int startOffset;
- private int endOffset;
-
- private int docBufferUpto;
- private int posBufferUpto;
-
- private Lucene50SkipReader skipper;
- private boolean skipped;
-
- final IndexInput startDocIn;
-
- IndexInput docIn;
- final IndexInput posIn;
- final IndexInput payIn;
- final BytesRef payload;
-
- final boolean indexHasOffsets;
- final boolean indexHasPayloads;
-
- private int docFreq; // number of docs in this posting list
- private long totalTermFreq; // number of positions in this posting list
- private int docUpto; // how many docs we've read
- private int doc; // doc we last read
- private int accum; // accumulator for doc deltas
- private int freq; // freq we last read
- private int position; // current position
-
- // how many positions "behind" we are; nextPosition must
- // skip these to "catch up":
- private int posPendingCount;
-
- // Lazy pos seek: if != -1 then we must seek to this FP
- // before reading positions:
- private long posPendingFP;
-
- // Lazy pay seek: if != -1 then we must seek to this FP
- // before reading payloads/offsets:
- private long payPendingFP;
-
- // Where this term's postings start in the .doc file:
- private long docTermStartFP;
-
- // Where this term's postings start in the .pos file:
- private long posTermStartFP;
-
- // Where this term's payloads/offsets start in the .pay
- // file:
- private long payTermStartFP;
-
- // File pointer where the last (vInt encoded) pos delta
- // block is. We need this to know whether to bulk
- // decode vs vInt decode the block:
- private long lastPosBlockFP;
-
- // Where this term's skip data starts (after
- // docTermStartFP) in the .doc file (or -1 if there is
- // no skip data for this term):
- private long skipOffset;
-
- private int nextSkipDoc;
-
- private boolean needsOffsets; // true if we actually need offsets
- private boolean needsPayloads; // true if we actually need payloads
- private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1
-
- public EverythingEnum(FieldInfo fieldInfo) throws IOException {
- indexHasOffsets =
- fieldInfo
- .getIndexOptions()
- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
- >= 0;
- indexHasPayloads = fieldInfo.hasPayloads();
-
- this.startDocIn = Lucene50PostingsReader.this.docIn;
- this.docIn = null;
- this.posIn = Lucene50PostingsReader.this.posIn.clone();
- if (indexHasOffsets || indexHasPayloads) {
- this.payIn = Lucene50PostingsReader.this.payIn.clone();
- } else {
- this.payIn = null;
- }
- encoded = new byte[MAX_ENCODED_SIZE];
- if (indexHasOffsets) {
- offsetStartDeltaBuffer = new int[MAX_DATA_SIZE];
- offsetLengthBuffer = new int[MAX_DATA_SIZE];
- } else {
- offsetStartDeltaBuffer = null;
- offsetLengthBuffer = null;
- startOffset = -1;
- endOffset = -1;
- }
-
- if (indexHasPayloads) {
- payloadLengthBuffer = new int[MAX_DATA_SIZE];
- payloadBytes = new byte[128];
- payload = new BytesRef();
- } else {
- payloadLengthBuffer = null;
- payloadBytes = null;
- payload = null;
- }
- }
-
- public boolean canReuse(IndexInput docIn, FieldInfo fieldInfo) {
- return docIn == startDocIn
- && indexHasOffsets
- == (fieldInfo
- .getIndexOptions()
- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
- >= 0)
- && indexHasPayloads == fieldInfo.hasPayloads();
- }
-
- public EverythingEnum reset(IntBlockTermState termState, int flags) throws IOException {
- docFreq = termState.docFreq;
- docTermStartFP = termState.docStartFP;
- posTermStartFP = termState.posStartFP;
- payTermStartFP = termState.payStartFP;
- skipOffset = termState.skipOffset;
- totalTermFreq = termState.totalTermFreq;
- singletonDocID = termState.singletonDocID;
- if (docFreq > 1) {
- if (docIn == null) {
- // lazy init
- docIn = startDocIn.clone();
- }
- docIn.seek(docTermStartFP);
- }
- posPendingFP = posTermStartFP;
- payPendingFP = payTermStartFP;
- posPendingCount = 0;
- if (termState.totalTermFreq < BLOCK_SIZE) {
- lastPosBlockFP = posTermStartFP;
- } else if (termState.totalTermFreq == BLOCK_SIZE) {
- lastPosBlockFP = -1;
- } else {
- lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset;
- }
-
- this.needsOffsets = PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS);
- this.needsPayloads = PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS);
-
- doc = -1;
- accum = 0;
- docUpto = 0;
- if (docFreq > BLOCK_SIZE) {
- nextSkipDoc = BLOCK_SIZE - 1; // we won't skip if target is found in first block
- } else {
- nextSkipDoc = NO_MORE_DOCS; // not enough docs for skipping
- }
- docBufferUpto = BLOCK_SIZE;
- skipped = false;
- return this;
- }
-
- @Override
- public int freq() throws IOException {
- return freq;
- }
-
- @Override
- public int docID() {
- return doc;
- }
-
- private void refillDocs() throws IOException {
- final int left = docFreq - docUpto;
- assert left > 0;
-
- if (left >= BLOCK_SIZE) {
- forUtil.readBlock(docIn, encoded, docDeltaBuffer);
- forUtil.readBlock(docIn, encoded, freqBuffer);
- } else if (docFreq == 1) {
- docDeltaBuffer[0] = singletonDocID;
- freqBuffer[0] = (int) totalTermFreq;
- } else {
- readVIntBlock(docIn, docDeltaBuffer, freqBuffer, left, true);
- }
- docBufferUpto = 0;
- }
-
- private void refillPositions() throws IOException {
- if (posIn.getFilePointer() == lastPosBlockFP) {
- final int count = (int) (totalTermFreq % BLOCK_SIZE);
- int payloadLength = 0;
- int offsetLength = 0;
- payloadByteUpto = 0;
- for (int i = 0; i < count; i++) {
- int code = posIn.readVInt();
- if (indexHasPayloads) {
- if ((code & 1) != 0) {
- payloadLength = posIn.readVInt();
- }
- payloadLengthBuffer[i] = payloadLength;
- posDeltaBuffer[i] = code >>> 1;
- if (payloadLength != 0) {
- if (payloadByteUpto + payloadLength > payloadBytes.length) {
- payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payloadLength);
- }
- posIn.readBytes(payloadBytes, payloadByteUpto, payloadLength);
- payloadByteUpto += payloadLength;
- }
- } else {
- posDeltaBuffer[i] = code;
- }
-
- if (indexHasOffsets) {
- int deltaCode = posIn.readVInt();
- if ((deltaCode & 1) != 0) {
- offsetLength = posIn.readVInt();
- }
- offsetStartDeltaBuffer[i] = deltaCode >>> 1;
- offsetLengthBuffer[i] = offsetLength;
- }
- }
- payloadByteUpto = 0;
- } else {
- forUtil.readBlock(posIn, encoded, posDeltaBuffer);
-
- if (indexHasPayloads) {
- if (needsPayloads) {
- forUtil.readBlock(payIn, encoded, payloadLengthBuffer);
- int numBytes = payIn.readVInt();
-
- if (numBytes > payloadBytes.length) {
- payloadBytes = ArrayUtil.grow(payloadBytes, numBytes);
- }
- payIn.readBytes(payloadBytes, 0, numBytes);
- } else {
- // this works, because when writing a vint block we always force the first length to be
- // written
- forUtil.skipBlock(payIn); // skip over lengths
- int numBytes = payIn.readVInt(); // read length of payloadBytes
- payIn.seek(payIn.getFilePointer() + numBytes); // skip over payloadBytes
- }
- payloadByteUpto = 0;
- }
-
- if (indexHasOffsets) {
- if (needsOffsets) {
- forUtil.readBlock(payIn, encoded, offsetStartDeltaBuffer);
- forUtil.readBlock(payIn, encoded, offsetLengthBuffer);
- } else {
- // this works, because when writing a vint block we always force the first length to be
- // written
- forUtil.skipBlock(payIn); // skip over starts
- forUtil.skipBlock(payIn); // skip over lengths
- }
- }
- }
- }
-
- @Override
- public int nextDoc() throws IOException {
- if (docUpto == docFreq) {
- return doc = NO_MORE_DOCS;
- }
- if (docBufferUpto == BLOCK_SIZE) {
- refillDocs();
- }
-
- accum += docDeltaBuffer[docBufferUpto];
- freq = freqBuffer[docBufferUpto];
- posPendingCount += freq;
- docBufferUpto++;
- docUpto++;
-
- doc = accum;
- position = 0;
- lastStartOffset = 0;
- return doc;
- }
-
- @Override
- public int advance(int target) throws IOException {
- // TODO: make frq block load lazy/skippable
-
- if (target > nextSkipDoc) {
- if (skipper == null) {
- // Lazy init: first time this enum has ever been used for skipping
- skipper =
- new Lucene50SkipReader(
- version, docIn.clone(), MAX_SKIP_LEVELS, true, indexHasOffsets, indexHasPayloads);
- }
-
- if (!skipped) {
- assert skipOffset != -1;
- // This is the first time this enum has skipped
- // since reset() was called; load the skip data:
- skipper.init(
- docTermStartFP + skipOffset, docTermStartFP, posTermStartFP, payTermStartFP, docFreq);
- skipped = true;
- }
-
- final int newDocUpto = skipper.skipTo(target) + 1;
-
- if (newDocUpto > docUpto) {
- // Skipper moved
- assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;
- docUpto = newDocUpto;
-
- // Force to read next block
- docBufferUpto = BLOCK_SIZE;
- accum = skipper.getDoc();
- docIn.seek(skipper.getDocPointer());
- posPendingFP = skipper.getPosPointer();
- payPendingFP = skipper.getPayPointer();
- posPendingCount = skipper.getPosBufferUpto();
- lastStartOffset = 0; // new document
- payloadByteUpto = skipper.getPayloadByteUpto();
- }
- nextSkipDoc = skipper.getNextSkipDoc();
- }
- if (docUpto == docFreq) {
- return doc = NO_MORE_DOCS;
- }
- if (docBufferUpto == BLOCK_SIZE) {
- refillDocs();
- }
-
- // Now scan:
- while (true) {
- accum += docDeltaBuffer[docBufferUpto];
- freq = freqBuffer[docBufferUpto];
- posPendingCount += freq;
- docBufferUpto++;
- docUpto++;
-
- if (accum >= target) {
- break;
- }
- if (docUpto == docFreq) {
- return doc = NO_MORE_DOCS;
- }
- }
-
- position = 0;
- lastStartOffset = 0;
- return doc = accum;
- }
-
- // TODO: in theory we could avoid loading frq block
- // when not needed, ie, use skip data to load how far to
- // seek the pos pointer ... instead of having to load frq
- // blocks only to sum up how many positions to skip
- private void skipPositions() throws IOException {
- // Skip positions now:
- int toSkip = posPendingCount - freq;
- // if (DEBUG) {
- // System.out.println(" FPR.skipPositions: toSkip=" + toSkip);
- // }
-
- final int leftInBlock = BLOCK_SIZE - posBufferUpto;
- if (toSkip < leftInBlock) {
- int end = posBufferUpto + toSkip;
- while (posBufferUpto < end) {
- if (indexHasPayloads) {
- payloadByteUpto += payloadLengthBuffer[posBufferUpto];
- }
- posBufferUpto++;
- }
- } else {
- toSkip -= leftInBlock;
- while (toSkip >= BLOCK_SIZE) {
- assert posIn.getFilePointer() != lastPosBlockFP;
- forUtil.skipBlock(posIn);
-
- if (indexHasPayloads) {
- // Skip payloadLength block:
- forUtil.skipBlock(payIn);
-
- // Skip payloadBytes block:
- int numBytes = payIn.readVInt();
- payIn.seek(payIn.getFilePointer() + numBytes);
- }
-
- if (indexHasOffsets) {
- forUtil.skipBlock(payIn);
- forUtil.skipBlock(payIn);
- }
- toSkip -= BLOCK_SIZE;
- }
- refillPositions();
- payloadByteUpto = 0;
- posBufferUpto = 0;
- while (posBufferUpto < toSkip) {
- if (indexHasPayloads) {
- payloadByteUpto += payloadLengthBuffer[posBufferUpto];
- }
- posBufferUpto++;
- }
- }
-
- position = 0;
- lastStartOffset = 0;
- }
-
- @Override
- public int nextPosition() throws IOException {
- assert posPendingCount > 0;
-
- if (posPendingFP != -1) {
- posIn.seek(posPendingFP);
- posPendingFP = -1;
-
- if (payPendingFP != -1 && payIn != null) {
- payIn.seek(payPendingFP);
- payPendingFP = -1;
- }
-
- // Force buffer refill:
- posBufferUpto = BLOCK_SIZE;
- }
-
- if (posPendingCount > freq) {
- skipPositions();
- posPendingCount = freq;
- }
-
- if (posBufferUpto == BLOCK_SIZE) {
- refillPositions();
- posBufferUpto = 0;
- }
- position += posDeltaBuffer[posBufferUpto];
-
- if (indexHasPayloads) {
- payloadLength = payloadLengthBuffer[posBufferUpto];
- payload.bytes = payloadBytes;
- payload.offset = payloadByteUpto;
- payload.length = payloadLength;
- payloadByteUpto += payloadLength;
- }
-
- if (indexHasOffsets) {
- startOffset = lastStartOffset + offsetStartDeltaBuffer[posBufferUpto];
- endOffset = startOffset + offsetLengthBuffer[posBufferUpto];
- lastStartOffset = startOffset;
- }
-
- posBufferUpto++;
- posPendingCount--;
- return position;
- }
-
- @Override
- public int startOffset() {
- return startOffset;
- }
-
- @Override
- public int endOffset() {
- return endOffset;
- }
-
- @Override
- public BytesRef getPayload() {
- if (payloadLength == 0) {
- return null;
- } else {
- return payload;
- }
- }
-
- @Override
- public long cost() {
- return docFreq;
- }
- }
-
- final class BlockImpactsPostingsEnum extends ImpactsEnum {
-
- private final byte[] encoded;
-
- private final int[] docDeltaBuffer = new int[MAX_DATA_SIZE];
- private final int[] freqBuffer = new int[MAX_DATA_SIZE];
- private final int[] posDeltaBuffer = new int[MAX_DATA_SIZE];
-
- private int docBufferUpto;
- private int posBufferUpto;
-
- private final Lucene50ScoreSkipReader skipper;
-
- final IndexInput docIn;
- final IndexInput posIn;
-
- final boolean indexHasOffsets;
- final boolean indexHasPayloads;
-
- private int docFreq; // number of docs in this posting list
- private long totalTermFreq; // number of positions in this posting list
- private int docUpto; // how many docs we've read
- private int doc; // doc we last read
- private int accum; // accumulator for doc deltas
- private int freq; // freq we last read
- private int position; // current position
-
- // how many positions "behind" we are; nextPosition must
- // skip these to "catch up":
- private int posPendingCount;
-
- // Lazy pos seek: if != -1 then we must seek to this FP
- // before reading positions:
- private long posPendingFP;
-
- // Where this term's postings start in the .doc file:
- private long docTermStartFP;
-
- // Where this term's postings start in the .pos file:
- private long posTermStartFP;
-
- // Where this term's payloads/offsets start in the .pay
- // file:
- private long payTermStartFP;
-
- // File pointer where the last (vInt encoded) pos delta
- // block is. We need this to know whether to bulk
- // decode vs vInt decode the block:
- private long lastPosBlockFP;
-
- private int nextSkipDoc = -1;
-
- private long seekTo = -1;
-
- public BlockImpactsPostingsEnum(FieldInfo fieldInfo, IntBlockTermState termState)
- throws IOException {
- indexHasOffsets =
- fieldInfo
- .getIndexOptions()
- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
- >= 0;
- indexHasPayloads = fieldInfo.hasPayloads();
-
- this.docIn = Lucene50PostingsReader.this.docIn.clone();
-
- encoded = new byte[MAX_ENCODED_SIZE];
-
- this.posIn = Lucene50PostingsReader.this.posIn.clone();
-
- docFreq = termState.docFreq;
- docTermStartFP = termState.docStartFP;
- posTermStartFP = termState.posStartFP;
- payTermStartFP = termState.payStartFP;
- totalTermFreq = termState.totalTermFreq;
- docIn.seek(docTermStartFP);
- posPendingFP = posTermStartFP;
- posPendingCount = 0;
- if (termState.totalTermFreq < BLOCK_SIZE) {
- lastPosBlockFP = posTermStartFP;
- } else if (termState.totalTermFreq == BLOCK_SIZE) {
- lastPosBlockFP = -1;
- } else {
- lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset;
- }
-
- doc = -1;
- accum = 0;
- docUpto = 0;
- docBufferUpto = BLOCK_SIZE;
-
- skipper =
- new Lucene50ScoreSkipReader(
- version, docIn.clone(), MAX_SKIP_LEVELS, true, indexHasOffsets, indexHasPayloads);
- skipper.init(
- docTermStartFP + termState.skipOffset,
- docTermStartFP,
- posTermStartFP,
- payTermStartFP,
- docFreq);
- }
-
- @Override
- public int freq() throws IOException {
- return freq;
- }
-
- @Override
- public int docID() {
- return doc;
- }
-
- private void refillDocs() throws IOException {
- final int left = docFreq - docUpto;
- assert left > 0;
-
- if (left >= BLOCK_SIZE) {
- forUtil.readBlock(docIn, encoded, docDeltaBuffer);
- forUtil.readBlock(docIn, encoded, freqBuffer);
- } else {
- readVIntBlock(docIn, docDeltaBuffer, freqBuffer, left, true);
- }
- docBufferUpto = 0;
- }
-
- private void refillPositions() throws IOException {
- if (posIn.getFilePointer() == lastPosBlockFP) {
- final int count = (int) (totalTermFreq % BLOCK_SIZE);
- int payloadLength = 0;
- for (int i = 0; i < count; i++) {
- int code = posIn.readVInt();
- if (indexHasPayloads) {
- if ((code & 1) != 0) {
- payloadLength = posIn.readVInt();
- }
- posDeltaBuffer[i] = code >>> 1;
- if (payloadLength != 0) {
- posIn.seek(posIn.getFilePointer() + payloadLength);
- }
- } else {
- posDeltaBuffer[i] = code;
- }
- if (indexHasOffsets) {
- if ((posIn.readVInt() & 1) != 0) {
- // offset length changed
- posIn.readVInt();
- }
- }
- }
- } else {
- forUtil.readBlock(posIn, encoded, posDeltaBuffer);
- }
- }
-
- @Override
- public void advanceShallow(int target) throws IOException {
- if (target > nextSkipDoc) {
- // always plus one to fix the result, since skip position in Lucene50SkipReader
- // is a little different from MultiLevelSkipListReader
- final int newDocUpto = skipper.skipTo(target) + 1;
-
- if (newDocUpto > docUpto) {
- // Skipper moved
- assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;
- docUpto = newDocUpto;
-
- // Force to read next block
- docBufferUpto = BLOCK_SIZE;
- accum = skipper.getDoc();
- posPendingFP = skipper.getPosPointer();
- posPendingCount = skipper.getPosBufferUpto();
- seekTo = skipper.getDocPointer(); // delay the seek
- }
- // next time we call advance, this is used to
- // foresee whether skipper is necessary.
- nextSkipDoc = skipper.getNextSkipDoc();
- }
- assert nextSkipDoc >= target;
- }
-
- @Override
- public Impacts getImpacts() throws IOException {
- advanceShallow(doc);
- return skipper.getImpacts();
- }
-
- @Override
- public int nextDoc() throws IOException {
- return advance(doc + 1);
- }
-
- @Override
- public int advance(int target) throws IOException {
- if (target > nextSkipDoc) {
- advanceShallow(target);
- }
- if (docUpto == docFreq) {
- return doc = NO_MORE_DOCS;
- }
- if (docBufferUpto == BLOCK_SIZE) {
- if (seekTo >= 0) {
- docIn.seek(seekTo);
- seekTo = -1;
- }
- refillDocs();
- }
-
- // Now scan:
- while (true) {
- accum += docDeltaBuffer[docBufferUpto];
- freq = freqBuffer[docBufferUpto];
- posPendingCount += freq;
- docBufferUpto++;
- docUpto++;
-
- if (accum >= target) {
- break;
- }
- if (docUpto == docFreq) {
- return doc = NO_MORE_DOCS;
- }
- }
- position = 0;
-
- return doc = accum;
- }
-
- // TODO: in theory we could avoid loading frq block
- // when not needed, ie, use skip data to load how far to
- // seek the pos pointer ... instead of having to load frq
- // blocks only to sum up how many positions to skip
- private void skipPositions() throws IOException {
- // Skip positions now:
- int toSkip = posPendingCount - freq;
-
- final int leftInBlock = BLOCK_SIZE - posBufferUpto;
- if (toSkip < leftInBlock) {
- posBufferUpto += toSkip;
- } else {
- toSkip -= leftInBlock;
- while (toSkip >= BLOCK_SIZE) {
- assert posIn.getFilePointer() != lastPosBlockFP;
- forUtil.skipBlock(posIn);
- toSkip -= BLOCK_SIZE;
- }
- refillPositions();
- posBufferUpto = toSkip;
- }
-
- position = 0;
- }
-
- @Override
- public int nextPosition() throws IOException {
- assert posPendingCount > 0;
-
- if (posPendingFP != -1) {
- posIn.seek(posPendingFP);
- posPendingFP = -1;
-
- // Force buffer refill:
- posBufferUpto = BLOCK_SIZE;
- }
-
- if (posPendingCount > freq) {
- skipPositions();
- posPendingCount = freq;
- }
-
- if (posBufferUpto == BLOCK_SIZE) {
- refillPositions();
- posBufferUpto = 0;
- }
- position += posDeltaBuffer[posBufferUpto++];
-
- posPendingCount--;
- return position;
- }
-
- @Override
- public int startOffset() {
- return -1;
- }
-
- @Override
- public int endOffset() {
- return -1;
- }
-
- @Override
- public BytesRef getPayload() {
- return null;
- }
-
- @Override
- public long cost() {
- return docFreq;
- }
- }
-
- final class BlockImpactsEverythingEnum extends ImpactsEnum {
-
- private final byte[] encoded;
-
- private final int[] docDeltaBuffer = new int[MAX_DATA_SIZE];
- private final int[] freqBuffer = new int[MAX_DATA_SIZE];
- private final int[] posDeltaBuffer = new int[MAX_DATA_SIZE];
-
- private final int[] payloadLengthBuffer;
- private final int[] offsetStartDeltaBuffer;
- private final int[] offsetLengthBuffer;
-
- private byte[] payloadBytes;
- private int payloadByteUpto;
- private int payloadLength;
-
- private int lastStartOffset;
- private int startOffset = -1;
- private int endOffset = -1;
-
- private int docBufferUpto;
- private int posBufferUpto;
-
- private final Lucene50ScoreSkipReader skipper;
-
- final IndexInput docIn;
- final IndexInput posIn;
- final IndexInput payIn;
- final BytesRef payload;
-
- final boolean indexHasFreq;
- final boolean indexHasPos;
- final boolean indexHasOffsets;
- final boolean indexHasPayloads;
-
- private int docFreq; // number of docs in this posting list
- private long totalTermFreq; // number of positions in this posting list
- private int docUpto; // how many docs we've read
- private int posDocUpTo; // for how many docs we've read positions, offsets, and payloads
- private int doc; // doc we last read
- private int accum; // accumulator for doc deltas
- private int position; // current position
-
- // how many positions "behind" we are; nextPosition must
- // skip these to "catch up":
- private int posPendingCount;
-
- // Lazy pos seek: if != -1 then we must seek to this FP
- // before reading positions:
- private long posPendingFP;
-
- // Lazy pay seek: if != -1 then we must seek to this FP
- // before reading payloads/offsets:
- private long payPendingFP;
-
- // Where this term's postings start in the .doc file:
- private long docTermStartFP;
-
- // Where this term's postings start in the .pos file:
- private long posTermStartFP;
-
- // Where this term's payloads/offsets start in the .pay
- // file:
- private long payTermStartFP;
-
- // File pointer where the last (vInt encoded) pos delta
- // block is. We need this to know whether to bulk
- // decode vs vInt decode the block:
- private long lastPosBlockFP;
-
- private int nextSkipDoc = -1;
-
- private final boolean needsPositions;
- private final boolean needsOffsets; // true if we actually need offsets
- private final boolean needsPayloads; // true if we actually need payloads
-
- private boolean
- isFreqsRead; // shows if freqs for the current doc block are read into freqBuffer
-
- private long seekTo = -1;
-
- public BlockImpactsEverythingEnum(FieldInfo fieldInfo, IntBlockTermState termState, int flags)
- throws IOException {
- indexHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
- indexHasPos =
- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- indexHasOffsets =
- fieldInfo
- .getIndexOptions()
- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
- >= 0;
- indexHasPayloads = fieldInfo.hasPayloads();
-
- needsPositions = PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS);
- needsOffsets = PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS);
- needsPayloads = PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS);
-
- this.docIn = Lucene50PostingsReader.this.docIn.clone();
-
- encoded = new byte[MAX_ENCODED_SIZE];
-
- if (indexHasPos && needsPositions) {
- this.posIn = Lucene50PostingsReader.this.posIn.clone();
- } else {
- this.posIn = null;
- }
-
- if ((indexHasOffsets && needsOffsets) || (indexHasPayloads && needsPayloads)) {
- this.payIn = Lucene50PostingsReader.this.payIn.clone();
- } else {
- this.payIn = null;
- }
-
- if (indexHasOffsets) {
- offsetStartDeltaBuffer = new int[MAX_DATA_SIZE];
- offsetLengthBuffer = new int[MAX_DATA_SIZE];
- } else {
- offsetStartDeltaBuffer = null;
- offsetLengthBuffer = null;
- startOffset = -1;
- endOffset = -1;
- }
-
- if (indexHasPayloads) {
- payloadLengthBuffer = new int[MAX_DATA_SIZE];
- payloadBytes = new byte[128];
- payload = new BytesRef();
- } else {
- payloadLengthBuffer = null;
- payloadBytes = null;
- payload = null;
- }
-
- docFreq = termState.docFreq;
- docTermStartFP = termState.docStartFP;
- posTermStartFP = termState.posStartFP;
- payTermStartFP = termState.payStartFP;
- totalTermFreq = termState.totalTermFreq;
- docIn.seek(docTermStartFP);
- posPendingFP = posTermStartFP;
- payPendingFP = payTermStartFP;
- posPendingCount = 0;
- if (termState.totalTermFreq < BLOCK_SIZE) {
- lastPosBlockFP = posTermStartFP;
- } else if (termState.totalTermFreq == BLOCK_SIZE) {
- lastPosBlockFP = -1;
- } else {
- lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset;
- }
-
- doc = -1;
- accum = 0;
- docUpto = 0;
- posDocUpTo = 0;
- isFreqsRead = true;
- docBufferUpto = BLOCK_SIZE;
-
- skipper =
- new Lucene50ScoreSkipReader(
- version,
- docIn.clone(),
- MAX_SKIP_LEVELS,
- indexHasPos,
- indexHasOffsets,
- indexHasPayloads);
- skipper.init(
- docTermStartFP + termState.skipOffset,
- docTermStartFP,
- posTermStartFP,
- payTermStartFP,
- docFreq);
-
- if (indexHasFreq == false) {
- Arrays.fill(freqBuffer, 1);
- }
- }
-
- @Override
- public int freq() throws IOException {
- if (indexHasFreq && (isFreqsRead == false)) {
- forUtil.readBlock(docIn, encoded, freqBuffer); // read freqs for this block
- isFreqsRead = true;
- }
- return freqBuffer[docBufferUpto - 1];
- }
-
- @Override
- public int docID() {
- return doc;
- }
-
- private void refillDocs() throws IOException {
- if (indexHasFreq) {
- if (isFreqsRead == false) { // previous freq block was not read
- // check if we need to load the previous freq block to catch up on positions or we can
- // skip it
- if (indexHasPos && needsPositions && (posDocUpTo < docUpto)) {
- forUtil.readBlock(docIn, encoded, freqBuffer); // load the previous freq block
- } else {
- forUtil.skipBlock(docIn); // skip it
- }
- isFreqsRead = true;
- }
- if (indexHasPos && needsPositions) {
- while (posDocUpTo
- < docUpto) { // catch on positions, bring posPendingCount upto the current doc
- posPendingCount += freqBuffer[docBufferUpto - (docUpto - posDocUpTo)];
- posDocUpTo++;
- }
- }
- }
-
- final int left = docFreq - docUpto;
- assert left > 0;
-
- if (left >= BLOCK_SIZE) {
- forUtil.readBlock(docIn, encoded, docDeltaBuffer);
- if (indexHasFreq) {
- isFreqsRead =
- false; // freq block will be loaded lazily when necessary, we don't load it here
- }
- } else {
- readVIntBlock(docIn, docDeltaBuffer, freqBuffer, left, indexHasFreq);
- }
- docBufferUpto = 0;
- }
-
- private void refillPositions() throws IOException {
- if (posIn.getFilePointer() == lastPosBlockFP) {
- final int count = (int) (totalTermFreq % BLOCK_SIZE);
- int payloadLength = 0;
- int offsetLength = 0;
- payloadByteUpto = 0;
- for (int i = 0; i < count; i++) {
- int code = posIn.readVInt();
- if (indexHasPayloads) {
- if ((code & 1) != 0) {
- payloadLength = posIn.readVInt();
- }
- payloadLengthBuffer[i] = payloadLength;
- posDeltaBuffer[i] = code >>> 1;
- if (payloadLength != 0) {
- if (payloadByteUpto + payloadLength > payloadBytes.length) {
- payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payloadLength);
- }
- posIn.readBytes(payloadBytes, payloadByteUpto, payloadLength);
- payloadByteUpto += payloadLength;
- }
- } else {
- posDeltaBuffer[i] = code;
- }
-
- if (indexHasOffsets) {
- int deltaCode = posIn.readVInt();
- if ((deltaCode & 1) != 0) {
- offsetLength = posIn.readVInt();
- }
- offsetStartDeltaBuffer[i] = deltaCode >>> 1;
- offsetLengthBuffer[i] = offsetLength;
- }
- }
- payloadByteUpto = 0;
- } else {
- forUtil.readBlock(posIn, encoded, posDeltaBuffer);
-
- if (indexHasPayloads && payIn != null) {
- if (needsPayloads) {
- forUtil.readBlock(payIn, encoded, payloadLengthBuffer);
- int numBytes = payIn.readVInt();
-
- if (numBytes > payloadBytes.length) {
- payloadBytes = ArrayUtil.grow(payloadBytes, numBytes);
- }
- payIn.readBytes(payloadBytes, 0, numBytes);
- } else {
- // this works, because when writing a vint block we always force the first length to be
- // written
- forUtil.skipBlock(payIn); // skip over lengths
- int numBytes = payIn.readVInt(); // read length of payloadBytes
- payIn.seek(payIn.getFilePointer() + numBytes); // skip over payloadBytes
- }
- payloadByteUpto = 0;
- }
-
- if (indexHasOffsets && payIn != null) {
- if (needsOffsets) {
- forUtil.readBlock(payIn, encoded, offsetStartDeltaBuffer);
- forUtil.readBlock(payIn, encoded, offsetLengthBuffer);
- } else {
- // this works, because when writing a vint block we always force the first length to be
- // written
- forUtil.skipBlock(payIn); // skip over starts
- forUtil.skipBlock(payIn); // skip over lengths
- }
- }
- }
- }
-
- @Override
- public void advanceShallow(int target) throws IOException {
- if (target > nextSkipDoc) {
- // always plus one to fix the result, since skip position in Lucene50SkipReader
- // is a little different from MultiLevelSkipListReader
- final int newDocUpto = skipper.skipTo(target) + 1;
-
- if (newDocUpto > docUpto) {
- // Skipper moved
- assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;
- docUpto = newDocUpto;
- posDocUpTo = docUpto;
-
- // Force to read next block
- docBufferUpto = BLOCK_SIZE;
- accum = skipper.getDoc();
- posPendingFP = skipper.getPosPointer();
- payPendingFP = skipper.getPayPointer();
- posPendingCount = skipper.getPosBufferUpto();
- lastStartOffset = 0; // new document
- payloadByteUpto = skipper.getPayloadByteUpto(); // actually, this is just lastSkipEntry
- seekTo = skipper.getDocPointer(); // delay the seek
- }
- // next time we call advance, this is used to
- // foresee whether skipper is necessary.
- nextSkipDoc = skipper.getNextSkipDoc();
- }
- assert nextSkipDoc >= target;
- }
-
- @Override
- public Impacts getImpacts() throws IOException {
- advanceShallow(doc);
- return skipper.getImpacts();
- }
-
- @Override
- public int nextDoc() throws IOException {
- return advance(doc + 1);
- }
-
- @Override
- public int advance(int target) throws IOException {
- if (target > nextSkipDoc) {
- advanceShallow(target);
- }
- if (docUpto == docFreq) {
- return doc = NO_MORE_DOCS;
- }
- if (docBufferUpto == BLOCK_SIZE) {
- if (seekTo >= 0) {
- docIn.seek(seekTo);
- seekTo = -1;
- isFreqsRead = true; // reset isFreqsRead
- }
- refillDocs();
- }
-
- // Now scan:
- while (true) {
- accum += docDeltaBuffer[docBufferUpto];
- docBufferUpto++;
- docUpto++;
-
- if (accum >= target) {
- break;
- }
- if (docUpto == docFreq) {
- return doc = NO_MORE_DOCS;
- }
- }
- position = 0;
- lastStartOffset = 0;
-
- return doc = accum;
- }
-
- // TODO: in theory we could avoid loading frq block
- // when not needed, ie, use skip data to load how far to
- // seek the pos pointer ... instead of having to load frq
- // blocks only to sum up how many positions to skip
- private void skipPositions() throws IOException {
- // Skip positions now:
- int toSkip = posPendingCount - freqBuffer[docBufferUpto - 1];
- // if (DEBUG) {
- // System.out.println(" FPR.skipPositions: toSkip=" + toSkip);
- // }
-
- final int leftInBlock = BLOCK_SIZE - posBufferUpto;
- if (toSkip < leftInBlock) {
- int end = posBufferUpto + toSkip;
- while (posBufferUpto < end) {
- if (indexHasPayloads) {
- payloadByteUpto += payloadLengthBuffer[posBufferUpto];
- }
- posBufferUpto++;
- }
- } else {
- toSkip -= leftInBlock;
- while (toSkip >= BLOCK_SIZE) {
- assert posIn.getFilePointer() != lastPosBlockFP;
- forUtil.skipBlock(posIn);
-
- if (indexHasPayloads && payIn != null) {
- // Skip payloadLength block:
- forUtil.skipBlock(payIn);
-
- // Skip payloadBytes block:
- int numBytes = payIn.readVInt();
- payIn.seek(payIn.getFilePointer() + numBytes);
- }
-
- if (indexHasOffsets && payIn != null) {
- forUtil.skipBlock(payIn);
- forUtil.skipBlock(payIn);
- }
- toSkip -= BLOCK_SIZE;
- }
- refillPositions();
- payloadByteUpto = 0;
- posBufferUpto = 0;
- while (posBufferUpto < toSkip) {
- if (indexHasPayloads) {
- payloadByteUpto += payloadLengthBuffer[posBufferUpto];
- }
- posBufferUpto++;
- }
- }
-
- position = 0;
- lastStartOffset = 0;
- }
-
- @Override
- public int nextPosition() throws IOException {
- if (indexHasPos == false || needsPositions == false) {
- return -1;
- }
-
- if (isFreqsRead == false) {
- forUtil.readBlock(docIn, encoded, freqBuffer); // read freqs for this docs block
- isFreqsRead = true;
- }
- while (posDocUpTo < docUpto) { // bring posPendingCount upto the current doc
- posPendingCount += freqBuffer[docBufferUpto - (docUpto - posDocUpTo)];
- posDocUpTo++;
- }
-
- assert posPendingCount > 0;
-
- if (posPendingFP != -1) {
- posIn.seek(posPendingFP);
- posPendingFP = -1;
-
- if (payPendingFP != -1 && payIn != null) {
- payIn.seek(payPendingFP);
- payPendingFP = -1;
- }
-
- // Force buffer refill:
- posBufferUpto = BLOCK_SIZE;
- }
-
- if (posPendingCount > freqBuffer[docBufferUpto - 1]) {
- skipPositions();
- posPendingCount = freqBuffer[docBufferUpto - 1];
- }
-
- if (posBufferUpto == BLOCK_SIZE) {
- refillPositions();
- posBufferUpto = 0;
- }
- position += posDeltaBuffer[posBufferUpto];
-
- if (indexHasPayloads) {
- payloadLength = payloadLengthBuffer[posBufferUpto];
- payload.bytes = payloadBytes;
- payload.offset = payloadByteUpto;
- payload.length = payloadLength;
- payloadByteUpto += payloadLength;
- }
-
- if (indexHasOffsets && needsOffsets) {
- startOffset = lastStartOffset + offsetStartDeltaBuffer[posBufferUpto];
- endOffset = startOffset + offsetLengthBuffer[posBufferUpto];
- lastStartOffset = startOffset;
- }
-
- posBufferUpto++;
- posPendingCount--;
- return position;
- }
-
- @Override
- public int startOffset() {
- return startOffset;
- }
-
- @Override
- public int endOffset() {
- return endOffset;
- }
-
- @Override
- public BytesRef getPayload() {
- if (payloadLength == 0) {
- return null;
- } else {
- return payload;
- }
- }
-
- @Override
- public long cost() {
- return docFreq;
- }
- }
-
- @Override
- public void checkIntegrity() throws IOException {
- if (docIn != null) {
- CodecUtil.checksumEntireFile(docIn);
- }
- if (posIn != null) {
- CodecUtil.checksumEntireFile(posIn);
- }
- if (payIn != null) {
- CodecUtil.checksumEntireFile(payIn);
- }
- }
-
- @Override
- public String toString() {
- return getClass().getSimpleName()
- + "(positions="
- + (posIn != null)
- + ",payloads="
- + (payIn != null)
- + ")";
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50ScoreSkipReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50ScoreSkipReader.java
deleted file mode 100644
index 11bdb595fc5..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50ScoreSkipReader.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import java.io.IOException;
-import java.util.Arrays;
-import org.apache.lucene.index.FreqAndNormBuffer;
-import org.apache.lucene.index.Impacts;
-import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.ArrayUtil;
-
-final class Lucene50ScoreSkipReader extends Lucene50SkipReader {
-
- private final byte[][] impactData;
- private final int[] impactDataLength;
- private final ByteArrayDataInput badi = new ByteArrayDataInput();
- private final Impacts impacts;
- private int numLevels = 1;
- private final FreqAndNormBuffer[] perLevelImpacts;
-
- public Lucene50ScoreSkipReader(
- int version,
- IndexInput skipStream,
- int maxSkipLevels,
- boolean hasPos,
- boolean hasOffsets,
- boolean hasPayloads) {
- super(version, skipStream, maxSkipLevels, hasPos, hasOffsets, hasPayloads);
- if (version < Lucene50PostingsFormat.VERSION_IMPACT_SKIP_DATA) {
- throw new IllegalStateException("Cannot skip based on scores if impacts are not indexed");
- }
- this.impactData = new byte[maxSkipLevels][];
- Arrays.fill(impactData, new byte[0]);
- this.impactDataLength = new int[maxSkipLevels];
- this.perLevelImpacts = new FreqAndNormBuffer[maxSkipLevels];
- for (int i = 0; i < perLevelImpacts.length; ++i) {
- perLevelImpacts[i] = new FreqAndNormBuffer();
- perLevelImpacts[i].add(Integer.MAX_VALUE, 1L);
- }
- impacts =
- new Impacts() {
-
- @Override
- public int numLevels() {
- return numLevels;
- }
-
- @Override
- public int getDocIdUpTo(int level) {
- return skipDoc[level];
- }
-
- @Override
- public FreqAndNormBuffer getImpacts(int level) {
- assert level < numLevels;
- if (impactDataLength[level] > 0) {
- badi.reset(impactData[level], 0, impactDataLength[level]);
- perLevelImpacts[level] = readImpacts(badi, perLevelImpacts[level]);
- impactDataLength[level] = 0;
- }
- return perLevelImpacts[level];
- }
- };
- }
-
- @Override
- public int skipTo(int target) throws IOException {
- int result = super.skipTo(target);
- if (numberOfSkipLevels > 0) {
- numLevels = numberOfSkipLevels;
- } else {
- // End of postings don't have skip data anymore, so we fill with dummy data
- // like SlowImpactsEnum.
- numLevels = 1;
- perLevelImpacts[0].size = 1;
- perLevelImpacts[0].freqs[0] = Integer.MAX_VALUE;
- perLevelImpacts[0].norms[0] = 1L;
- impactDataLength[0] = 0;
- }
- return result;
- }
-
- Impacts getImpacts() {
- return impacts;
- }
-
- @Override
- protected void readImpacts(int level, IndexInput skipStream) throws IOException {
- int length = skipStream.readVInt();
- if (impactData[level].length < length) {
- impactData[level] = new byte[ArrayUtil.oversize(length, Byte.BYTES)];
- }
- skipStream.readBytes(impactData[level], 0, length);
- impactDataLength[level] = length;
- }
-
- static FreqAndNormBuffer readImpacts(ByteArrayDataInput in, FreqAndNormBuffer reuse) {
- int maxNumImpacts = in.length(); // at most one impact per byte
- reuse.growNoCopy(maxNumImpacts);
-
- int freq = 0;
- long norm = 0;
- int size = 0;
- while (in.getPosition() < in.length()) {
- int freqDelta = in.readVInt();
- if ((freqDelta & 0x01) != 0) {
- freq += 1 + (freqDelta >>> 1);
- try {
- norm += 1 + in.readZLong();
- } catch (IOException e) {
- throw new RuntimeException(e); // cannot happen on a BADI
- }
- } else {
- freq += 1 + (freqDelta >>> 1);
- norm++;
- }
- reuse.freqs[size] = freq;
- reuse.norms[size] = norm;
- size++;
- }
- reuse.size = size;
- return reuse;
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50SkipReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50SkipReader.java
deleted file mode 100644
index a74eacdd21d..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50SkipReader.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.BLOCK_SIZE;
-
-import java.io.IOException;
-import java.util.Arrays;
-import org.apache.lucene.codecs.MultiLevelSkipListReader;
-import org.apache.lucene.store.IndexInput;
-
-/**
- * Implements the skip list reader for block postings format that stores positions and payloads.
- *
- * <p>Although this skipper uses MultiLevelSkipListReader as an interface, its definition of skip
- * position will be a little different.
- *
- * <p>For example, when skipInterval = blockSize = 3, df = 2*skipInterval = 6,
- *
- * <pre>
- * 0 1 2 3 4 5
- * d d d d d d (posting list)
- * ^ ^ (skip point in MultiLeveSkipWriter)
- * ^ (skip point in Lucene50SkipWriter)
- * </pre>
- *
- * In this case, MultiLevelSkipListReader will use the last document as a skip point, while
- * Lucene50SkipReader should assume no skip point will comes.
- *
- * <p>If we use the interface directly in Lucene50SkipReader, it may silly try to read another skip
- * data after the only skip point is loaded.
- *
- * <p>To illustrate this, we can call skipTo(d[5]), since skip point d[3] has smaller docId, and
- * numSkipped+blockSize== df, the MultiLevelSkipListReader will assume the skip list isn't exhausted
- * yet, and try to load a non-existed skip point
- *
- * <p>Therefore, we'll trim df before passing it to the interface. see trim(int)
- */
-class Lucene50SkipReader extends MultiLevelSkipListReader {
- private final int version;
- private long[] docPointer;
- private long[] posPointer;
- private long[] payPointer;
- private int[] posBufferUpto;
- private int[] payloadByteUpto;
-
- private long lastPosPointer;
- private long lastPayPointer;
- private int lastPayloadByteUpto;
- private long lastDocPointer;
- private int lastPosBufferUpto;
-
- public Lucene50SkipReader(
- int version,
- IndexInput skipStream,
- int maxSkipLevels,
- boolean hasPos,
- boolean hasOffsets,
- boolean hasPayloads) {
- super(skipStream, maxSkipLevels, BLOCK_SIZE, 8);
- this.version = version;
- docPointer = new long[maxSkipLevels];
- if (hasPos) {
- posPointer = new long[maxSkipLevels];
- posBufferUpto = new int[maxSkipLevels];
- if (hasPayloads) {
- payloadByteUpto = new int[maxSkipLevels];
- } else {
- payloadByteUpto = null;
- }
- if (hasOffsets || hasPayloads) {
- payPointer = new long[maxSkipLevels];
- } else {
- payPointer = null;
- }
- } else {
- posPointer = null;
- }
- }
-
- /**
- * Trim original docFreq to tell skipReader read proper number of skip points.
- *
- * <p>Since our definition in Lucene50Skip* is a little different from MultiLevelSkip* This
- * trimmed docFreq will prevent skipReader from: 1. silly reading a non-existed skip point after
- * the last block boundary 2. moving into the vInt block
- */
- protected int trim(int df) {
- return df % BLOCK_SIZE == 0 ? df - 1 : df;
- }
-
- public void init(
- long skipPointer, long docBasePointer, long posBasePointer, long payBasePointer, int df)
- throws IOException {
- super.init(skipPointer, trim(df));
- lastDocPointer = docBasePointer;
- lastPosPointer = posBasePointer;
- lastPayPointer = payBasePointer;
-
- Arrays.fill(docPointer, docBasePointer);
- if (posPointer != null) {
- Arrays.fill(posPointer, posBasePointer);
- if (payPointer != null) {
- Arrays.fill(payPointer, payBasePointer);
- }
- } else {
- assert posBasePointer == 0;
- }
- }
-
- /**
- * Returns the doc pointer of the doc to which the last call of {@link
- * MultiLevelSkipListReader#skipTo(int)} has skipped.
- */
- public long getDocPointer() {
- return lastDocPointer;
- }
-
- public long getPosPointer() {
- return lastPosPointer;
- }
-
- public int getPosBufferUpto() {
- return lastPosBufferUpto;
- }
-
- public long getPayPointer() {
- return lastPayPointer;
- }
-
- public int getPayloadByteUpto() {
- return lastPayloadByteUpto;
- }
-
- public int getNextSkipDoc() {
- return skipDoc[0];
- }
-
- @Override
- protected void seekChild(int level) throws IOException {
- super.seekChild(level);
- docPointer[level] = lastDocPointer;
- if (posPointer != null) {
- posPointer[level] = lastPosPointer;
- posBufferUpto[level] = lastPosBufferUpto;
- if (payloadByteUpto != null) {
- payloadByteUpto[level] = lastPayloadByteUpto;
- }
- if (payPointer != null) {
- payPointer[level] = lastPayPointer;
- }
- }
- }
-
- @Override
- protected void setLastSkipData(int level) {
- super.setLastSkipData(level);
- lastDocPointer = docPointer[level];
-
- if (posPointer != null) {
- lastPosPointer = posPointer[level];
- lastPosBufferUpto = posBufferUpto[level];
- if (payPointer != null) {
- lastPayPointer = payPointer[level];
- }
- if (payloadByteUpto != null) {
- lastPayloadByteUpto = payloadByteUpto[level];
- }
- }
- }
-
- @Override
- protected int readSkipData(int level, IndexInput skipStream) throws IOException {
- int delta = skipStream.readVInt();
- docPointer[level] += skipStream.readVLong();
-
- if (posPointer != null) {
- posPointer[level] += skipStream.readVLong();
- posBufferUpto[level] = skipStream.readVInt();
-
- if (payloadByteUpto != null) {
- payloadByteUpto[level] = skipStream.readVInt();
- }
-
- if (payPointer != null) {
- payPointer[level] += skipStream.readVLong();
- }
- }
- readImpacts(level, skipStream);
- return delta;
- }
-
- // The default impl skips impacts
- protected void readImpacts(int level, IndexInput skipStream) throws IOException {
- if (version >= Lucene50PostingsFormat.VERSION_IMPACT_SKIP_DATA) {
- // The base implementation skips impacts, they are not used
- skipStream.skipBytes(skipStream.readVInt());
- }
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50StoredFieldsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50StoredFieldsFormat.java
deleted file mode 100644
index 560bdb2d1f3..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50StoredFieldsFormat.java
+++ /dev/null
@@ -1,235 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import java.io.IOException;
-import java.util.Objects;
-import org.apache.lucene.backward_codecs.compressing.CompressionMode;
-import org.apache.lucene.backward_codecs.compressing.Compressor;
-import org.apache.lucene.backward_codecs.compressing.Decompressor;
-import org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsFormat;
-import org.apache.lucene.backward_codecs.packed.LegacyDirectMonotonicWriter;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.StoredFieldsFormat;
-import org.apache.lucene.codecs.StoredFieldsReader;
-import org.apache.lucene.codecs.StoredFieldsWriter;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.StoredFieldVisitor;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.compress.LZ4;
-
-/**
- * Lucene 5.0 stored fields format.
- *
- * <p><b>Principle</b>
- *
- * <p>This {@link StoredFieldsFormat} compresses blocks of documents in order to improve the
- * compression ratio compared to document-level compression. It uses the <a
- * href="http://code.google.com/p/lz4/">LZ4</a> compression algorithm by default in 16KB blocks,
- * which is fast to compress and very fast to decompress data. Although the default compression
- * method that is used ({@link Mode#BEST_SPEED BEST_SPEED}) focuses more on speed than on
- * compression ratio, it should provide interesting compression ratios for redundant inputs (such as
- * log files, HTML or plain text). For higher compression, you can choose ({@link
- * Mode#BEST_COMPRESSION BEST_COMPRESSION}), which uses the <a
- * href="http://en.wikipedia.org/wiki/DEFLATE">DEFLATE</a> algorithm with 60KB blocks for a better
- * ratio at the expense of slower performance. These two options can be configured like this:
- *
- * <pre><code class="language-java">
- * // the default: for high performance
- * indexWriterConfig.setCodec(new Lucene54Codec(Mode.BEST_SPEED));
- * // instead for higher performance (but slower):
- * // indexWriterConfig.setCodec(new Lucene54Codec(Mode.BEST_COMPRESSION));
- * </code></pre>
- *
- * <p><b>File formats</b>
- *
- * <p>Stored fields are represented by three files:
- *
- * <ol>
- * <li><a id="field_data"></a>
- * <p>A fields data file (extension <code>.fdt</code>). This file stores a compact
- * representation of documents in compressed blocks of 16KB or more. When writing a segment,
- * documents are appended to an in-memory <code>byte[]</code> buffer. When its size reaches
- * 16KB or more, some metadata about the documents is flushed to disk, immediately followed by
- * a compressed representation of the buffer using the <a
- * href="https://github.com/lz4/lz4">LZ4</a> <a
- * href="http://fastcompression.blogspot.fr/2011/05/lz4-explained.html">compression
- * format</a>.
- * <p>Notes
- * <ul>
- * <li>If documents are larger than 16KB then chunks will likely contain only one document.
- * However, documents can never spread across several chunks (all fields of a single
- * document are in the same chunk).
- * <li>When at least one document in a chunk is large enough so that the chunk is larger
- * than 32KB, the chunk will actually be compressed in several LZ4 blocks of 16KB. This
- * allows {@link StoredFieldVisitor}s which are only interested in the first fields of a
- * document to not have to decompress 10MB of data if the document is 10MB, but only
- * 16KB.
- * <li>Given that the original lengths are written in the metadata of the chunk, the
- * decompressor can leverage this information to stop decoding as soon as enough data
- * has been decompressed.
- * <li>In case documents are incompressible, the overhead of the compression format is less
- * than 0.5%.
- * </ul>
- * <li><a id="field_index"></a>
- * <p>A fields index file (extension <code>.fdx</code>). This file stores two {@link
- * LegacyDirectMonotonicWriter monotonic arrays}, one for the first doc IDs of each block of
- * compressed documents, and another one for the corresponding offsets on disk. At search
- * time, the array containing doc IDs is binary-searched in order to find the block that
- * contains the expected doc ID, and the associated offset on disk is retrieved from the
- * second array.
- * <li><a id="field_meta"></a>
- * <p>A fields meta file (extension <code>.fdm</code>). This file stores metadata about the
- * monotonic arrays stored in the index file.
- * </ol>
- *
- * <p><b>Known limitations</b>
- *
- * <p>This {@link StoredFieldsFormat} does not support individual documents larger than (<code>
- * 2<sup>31</sup> - 2<sup>14</sup></code>) bytes.
- *
- * @lucene.experimental
- */
-public class Lucene50StoredFieldsFormat extends StoredFieldsFormat {
-
- /** Configuration option for stored fields. */
- public enum Mode {
- /** Trade compression ratio for retrieval speed. */
- BEST_SPEED,
- /** Trade retrieval speed for compression ratio. */
- BEST_COMPRESSION
- }
-
- /** Attribute key for compression mode. */
- public static final String MODE_KEY = Lucene50StoredFieldsFormat.class.getSimpleName() + ".mode";
-
- final Mode mode;
-
- /** Stored fields format with default options */
- public Lucene50StoredFieldsFormat() {
- this(Mode.BEST_SPEED);
- }
-
- /** Stored fields format with specified mode */
- public Lucene50StoredFieldsFormat(Mode mode) {
- this.mode = Objects.requireNonNull(mode);
- }
-
- @Override
- public final StoredFieldsReader fieldsReader(
- Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
- String value = si.getAttribute(MODE_KEY);
- if (value == null) {
- throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name);
- }
- Mode mode = Mode.valueOf(value);
- return impl(mode).fieldsReader(directory, si, fn, context);
- }
-
- @Override
- public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context)
- throws IOException {
- throw new UnsupportedOperationException("Old codecs may only be used for reading");
- }
-
- StoredFieldsFormat impl(Mode mode) {
- switch (mode) {
- case BEST_SPEED:
- return new Lucene50CompressingStoredFieldsFormat(
- "Lucene50StoredFieldsFastData", FAST_MODE, 1 << 14, 128, 10);
- case BEST_COMPRESSION:
- return new Lucene50CompressingStoredFieldsFormat(
- "Lucene50StoredFieldsHighData", CompressionMode.HIGH_COMPRESSION, 61440, 512, 10);
- default:
- throw new AssertionError();
- }
- }
-
- static final CompressionMode FAST_MODE =
- new CompressionMode() {
-
- @Override
- public Compressor newCompressor() {
- return new LZ4FastCompressor();
- }
-
- @Override
- public Decompressor newDecompressor() {
- return LZ4_DECOMPRESSOR;
- }
-
- @Override
- public String toString() {
- return "FAST";
- }
- };
-
- private static final class LZ4FastCompressor extends Compressor {
-
- private final LZ4.FastCompressionHashTable ht;
-
- LZ4FastCompressor() {
- ht = new LZ4.FastCompressionHashTable();
- }
-
- @Override
- public void compress(byte[] bytes, int off, int len, DataOutput out) throws IOException {
- LZ4.compress(bytes, off, len, EndiannessReverserUtil.wrapDataOutput(out), ht);
- }
-
- @Override
- public void close() throws IOException {
- // no-op
- }
- }
-
- private static final Decompressor LZ4_DECOMPRESSOR =
- new Decompressor() {
-
- @Override
- public void decompress(
- DataInput in, int originalLength, int offset, int length, BytesRef bytes)
- throws IOException {
- assert offset + length <= originalLength;
- // add 7 padding bytes, this is not necessary but can help decompression run faster
- if (bytes.bytes.length < originalLength + 7) {
- bytes.bytes = new byte[ArrayUtil.oversize(originalLength + 7, 1)];
- }
- final int decompressedLength =
- LZ4.decompress(
- EndiannessReverserUtil.wrapDataInput(in), offset + length, bytes.bytes, 0);
- if (decompressedLength > originalLength) {
- throw new CorruptIndexException(
- "Corrupted: lengths mismatch: " + decompressedLength + " > " + originalLength, in);
- }
- bytes.offset = offset;
- bytes.length = length;
- }
-
- @Override
- public Decompressor clone() {
- return this;
- }
- };
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50TermVectorsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50TermVectorsFormat.java
deleted file mode 100644
index 5a37dc450af..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50TermVectorsFormat.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsFormat;
-import org.apache.lucene.backward_codecs.lucene87.Lucene87StoredFieldsFormat;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.util.packed.BlockPackedWriter;
-import org.apache.lucene.util.packed.PackedInts;
-
-/**
- * Lucene 5.0 {@link TermVectorsFormat term vectors format}.
- *
- * <p>Very similarly to {@link Lucene87StoredFieldsFormat}, this format is based on compressed
- * chunks of data, with document-level granularity so that a document can never span across distinct
- * chunks. Moreover, data is made as compact as possible:
- *
- * <ul>
- * <li>textual data is compressed using the very light, <a
- * href="http://code.google.com/p/lz4/">LZ4</a> compression algorithm,
- * <li>binary data is written using fixed-size blocks of {@link PackedInts packed ints}.
- * </ul>
- *
- * <p>Term vectors are stored using two files
- *
- * <ul>
- * <li>a data file where terms, frequencies, positions, offsets and payloads are stored,
- * <li>an index file, loaded into memory, used to locate specific documents in the data file.
- * </ul>
- *
- * Looking up term vectors for any document requires at most 1 disk seek.
- *
- * <p><b>File formats</b>
- *
- * <ol>
- * <li><a id="vector_data"></a>
- * <p>A vector data file (extension <code>.tvd</code>). This file stores terms, frequencies,
- * positions, offsets and payloads for every document. Upon writing a new segment, it
- * accumulates data into memory until the buffer used to store terms and payloads grows beyond
- * 4KB. Then it flushes all metadata, terms and positions to disk using <a
- * href="http://code.google.com/p/lz4/">LZ4</a> compression for terms and payloads and {@link
- * BlockPackedWriter blocks of packed ints} for positions.
- * <p>Here is a more detailed description of the field data file format:
- * <ul>
- * <li>VectorData (.tvd) --&gt; &lt;Header&gt;, PackedIntsVersion, ChunkSize,
- * &lt;Chunk&gt;<sup>ChunkCount</sup>, ChunkCount, DirtyChunkCount, Footer
- * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
- * <li>PackedIntsVersion --&gt; {@link PackedInts#VERSION_CURRENT} as a {@link
- * DataOutput#writeVInt VInt}
- * <li>ChunkSize is the number of bytes of terms to accumulate before flushing, as a {@link
- * DataOutput#writeVInt VInt}
- * <li>ChunkCount is not known in advance and is the number of chunks necessary to store all
- * document of the segment
- * <li>Chunk --&gt; DocBase, ChunkDocs, &lt; NumFields &gt;, &lt; FieldNums &gt;, &lt;
- * FieldNumOffs &gt;, &lt; Flags &gt;, &lt; NumTerms &gt;, &lt; TermLengths &gt;, &lt;
- * TermFreqs &gt;, &lt; Positions &gt;, &lt; StartOffsets &gt;, &lt; Lengths &gt;, &lt;
- * PayloadLengths &gt;, &lt; TermAndPayloads &gt;
- * <li>DocBase is the ID of the first doc of the chunk as a {@link DataOutput#writeVInt
- * VInt}
- * <li>ChunkDocs is the number of documents in the chunk
- * <li>NumFields --&gt; DocNumFields<sup>ChunkDocs</sup>
- * <li>DocNumFields is the number of fields for each doc, written as a {@link
- * DataOutput#writeVInt VInt} if ChunkDocs==1 and as a {@link PackedInts} array
- * otherwise
- * <li>FieldNums --&gt; FieldNumDelta<sup>TotalDistincFields</sup>, a delta-encoded list of
- * the sorted unique field numbers present in the chunk
- * <li>FieldNumOffs --&gt; FieldNumOff<sup>TotalFields</sup>, as a {@link PackedInts} array
- * <li>FieldNumOff is the offset of the field number in FieldNums
- * <li>TotalFields is the total number of fields (sum of the values of NumFields)
- * <li>Flags --&gt; Bit &lt; FieldFlags &gt;
- * <li>Bit is a single bit which when true means that fields have the same options for every
- * document in the chunk
- * <li>FieldFlags --&gt; if Bit==1: Flag<sup>TotalDistinctFields</sup> else
- * Flag<sup>TotalFields</sup>
- * <li>Flag: a 3-bits int where:
- * <ul>
- * <li>the first bit means that the field has positions
- * <li>the second bit means that the field has offsets
- * <li>the third bit means that the field has payloads
- * </ul>
- * <li>NumTerms --&gt; FieldNumTerms<sup>TotalFields</sup>
- * <li>FieldNumTerms: the number of terms for each field, using {@link BlockPackedWriter
- * blocks of 64 packed ints}
- * <li>TermLengths --&gt; PrefixLength<sup>TotalTerms</sup>
- * SuffixLength<sup>TotalTerms</sup>
- * <li>TotalTerms: total number of terms (sum of NumTerms)
- * <li>PrefixLength: 0 for the first term of a field, the common prefix with the previous
- * term otherwise using {@link BlockPackedWriter blocks of 64 packed ints}
- * <li>SuffixLength: length of the term minus PrefixLength for every term using {@link
- * BlockPackedWriter blocks of 64 packed ints}
- * <li>TermFreqs --&gt; TermFreqMinus1<sup>TotalTerms</sup>
- * <li>TermFreqMinus1: (frequency - 1) for each term using {@link BlockPackedWriter blocks
- * of 64 packed ints}
- * <li>Positions --&gt; PositionDelta<sup>TotalPositions</sup>
- * <li>TotalPositions is the sum of frequencies of terms of all fields that have positions
- * <li>PositionDelta: the absolute position for the first position of a term, and the
- * difference with the previous positions for following positions using {@link
- * BlockPackedWriter blocks of 64 packed ints}
- * <li>StartOffsets --&gt; (AvgCharsPerTerm<sup>TotalDistinctFields</sup>)
- * StartOffsetDelta<sup>TotalOffsets</sup>
- * <li>TotalOffsets is the sum of frequencies of terms of all fields that have offsets
- * <li>AvgCharsPerTerm: average number of chars per term, encoded as a float on 4 bytes.
- * They are not present if no field has both positions and offsets enabled.
- * <li>StartOffsetDelta: (startOffset - previousStartOffset - AvgCharsPerTerm *
- * PositionDelta). previousStartOffset is 0 for the first offset and AvgCharsPerTerm is
- * 0 if the field has no positions using {@link BlockPackedWriter blocks of 64 packed
- * ints}
- * <li>Lengths --&gt; LengthMinusTermLength<sup>TotalOffsets</sup>
- * <li>LengthMinusTermLength: (endOffset - startOffset - termLength) using {@link
- * BlockPackedWriter blocks of 64 packed ints}
- * <li>PayloadLengths --&gt; PayloadLength<sup>TotalPayloads</sup>
- * <li>TotalPayloads is the sum of frequencies of terms of all fields that have payloads
- * <li>PayloadLength is the payload length encoded using {@link BlockPackedWriter blocks of
- * 64 packed ints}
- * <li>TermAndPayloads --&gt; LZ4-compressed representation of &lt; FieldTermsAndPayLoads
- * &gt;<sup>TotalFields</sup>
- * <li>FieldTermsAndPayLoads --&gt; Terms (Payloads)
- * <li>Terms: term bytes
- * <li>Payloads: payload bytes (if the field has payloads)
- * <li>ChunkCount --&gt; the number of chunks in this file
- * <li>DirtyChunkCount --&gt; the number of prematurely flushed chunks in this file
- * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
- * </ul>
- * <li><a id="vector_index"></a>
- * <p>An index file (extension <code>.tvx</code>).
- * <ul>
- * <li>VectorIndex (.tvx) --&gt; &lt;Header&gt;, &lt;ChunkIndex&gt;, Footer
- * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
- * <li>ChunkIndex: See FieldsIndexWriter
- * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
- * </ul>
- * </ol>
- *
- * @lucene.experimental
- */
-public final class Lucene50TermVectorsFormat extends Lucene50CompressingTermVectorsFormat {
-
- /** Sole constructor. */
- public Lucene50TermVectorsFormat() {
- super("Lucene50TermVectorsData", "", Lucene50StoredFieldsFormat.FAST_MODE, 1 << 12, 128, 10);
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndex.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndex.java
deleted file mode 100644
index 4a4eba771fa..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndex.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50.compressing;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-abstract class FieldsIndex implements Cloneable, Closeable {
-
- /** Get the start pointer for the block that contains the given docID. */
- abstract long getStartPointer(int docID);
-
- /** Check the integrity of the index. */
- abstract void checkIntegrity() throws IOException;
-
- @Override
- public abstract FieldsIndex clone();
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndexReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndexReader.java
deleted file mode 100644
index 9994948795c..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndexReader.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50.compressing;
-
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.util.Objects;
-import org.apache.lucene.backward_codecs.packed.LegacyDirectMonotonicReader;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.RandomAccessInput;
-import org.apache.lucene.util.IOUtils;
-
-final class FieldsIndexReader extends FieldsIndex {
-
- static final int VERSION_START = 0;
- static final int VERSION_CURRENT = 0;
-
- private final int maxDoc;
- private final int blockShift;
- private final int numChunks;
- private final LegacyDirectMonotonicReader.Meta docsMeta;
- private final LegacyDirectMonotonicReader.Meta startPointersMeta;
- private final IndexInput indexInput;
- private final long docsStartPointer,
- docsEndPointer,
- startPointersStartPointer,
- startPointersEndPointer;
- private final LegacyDirectMonotonicReader docs, startPointers;
- private final long maxPointer;
-
- FieldsIndexReader(
- Directory dir,
- String name,
- String suffix,
- String extension,
- String codecName,
- byte[] id,
- IndexInput metaIn)
- throws IOException {
- maxDoc = metaIn.readInt();
- blockShift = metaIn.readInt();
- numChunks = metaIn.readInt();
- docsStartPointer = metaIn.readLong();
- docsMeta = LegacyDirectMonotonicReader.loadMeta(metaIn, numChunks, blockShift);
- docsEndPointer = startPointersStartPointer = metaIn.readLong();
- startPointersMeta = LegacyDirectMonotonicReader.loadMeta(metaIn, numChunks, blockShift);
- startPointersEndPointer = metaIn.readLong();
- maxPointer = metaIn.readLong();
-
- indexInput =
- EndiannessReverserUtil.openInput(
- dir, IndexFileNames.segmentFileName(name, suffix, extension), IOContext.DEFAULT);
- try {
- CodecUtil.checkIndexHeader(
- indexInput, codecName + "Idx", VERSION_START, VERSION_CURRENT, id, suffix);
- CodecUtil.retrieveChecksum(indexInput);
- } catch (Throwable t) {
- IOUtils.closeWhileSuppressingExceptions(t, indexInput);
- throw t;
- }
- final RandomAccessInput docsSlice =
- indexInput.randomAccessSlice(docsStartPointer, docsEndPointer - docsStartPointer);
- final RandomAccessInput startPointersSlice =
- indexInput.randomAccessSlice(
- startPointersStartPointer, startPointersEndPointer - startPointersStartPointer);
- docs = LegacyDirectMonotonicReader.getInstance(docsMeta, docsSlice);
- startPointers = LegacyDirectMonotonicReader.getInstance(startPointersMeta, startPointersSlice);
- }
-
- private FieldsIndexReader(FieldsIndexReader other) throws IOException {
- maxDoc = other.maxDoc;
- numChunks = other.numChunks;
- blockShift = other.blockShift;
- docsMeta = other.docsMeta;
- startPointersMeta = other.startPointersMeta;
- indexInput = other.indexInput.clone();
- docsStartPointer = other.docsStartPointer;
- docsEndPointer = other.docsEndPointer;
- startPointersStartPointer = other.startPointersStartPointer;
- startPointersEndPointer = other.startPointersEndPointer;
- maxPointer = other.maxPointer;
- final RandomAccessInput docsSlice =
- indexInput.randomAccessSlice(docsStartPointer, docsEndPointer - docsStartPointer);
- final RandomAccessInput startPointersSlice =
- indexInput.randomAccessSlice(
- startPointersStartPointer, startPointersEndPointer - startPointersStartPointer);
- docs = LegacyDirectMonotonicReader.getInstance(docsMeta, docsSlice);
- startPointers = LegacyDirectMonotonicReader.getInstance(startPointersMeta, startPointersSlice);
- }
-
- @Override
- public void close() throws IOException {
- indexInput.close();
- }
-
- @Override
- long getStartPointer(int docID) {
- Objects.checkIndex(docID, maxDoc);
- long blockIndex = docs.binarySearch(0, numChunks, docID);
- if (blockIndex < 0) {
- blockIndex = -2 - blockIndex;
- }
- return startPointers.get(blockIndex);
- }
-
- @Override
- public FieldsIndex clone() {
- try {
- return new FieldsIndexReader(this);
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- }
-
- public long getMaxPointer() {
- return maxPointer;
- }
-
- @Override
- void checkIntegrity() throws IOException {
- CodecUtil.checksumEntireFile(indexInput);
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/LegacyFieldsIndexReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/LegacyFieldsIndexReader.java
deleted file mode 100644
index 2aba9e05332..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/LegacyFieldsIndexReader.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50.compressing;
-
-import static org.apache.lucene.util.BitUtil.zigZagDecode;
-
-import java.io.IOException;
-import org.apache.lucene.backward_codecs.packed.LegacyPackedInts;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.packed.PackedInts;
-
-final class LegacyFieldsIndexReader extends FieldsIndex {
-
- final int maxDoc;
- final int[] docBases;
- final long[] startPointers;
- final int[] avgChunkDocs;
- final long[] avgChunkSizes;
- final PackedInts.Reader[] docBasesDeltas; // delta from the avg
- final PackedInts.Reader[] startPointersDeltas; // delta from the avg
-
- // It is the responsibility of the caller to close fieldsIndexIn after this constructor
- // has been called
- LegacyFieldsIndexReader(IndexInput fieldsIndexIn, SegmentInfo si) throws IOException {
- maxDoc = si.maxDoc();
- int[] docBases = new int[16];
- long[] startPointers = new long[16];
- int[] avgChunkDocs = new int[16];
- long[] avgChunkSizes = new long[16];
- PackedInts.Reader[] docBasesDeltas = new PackedInts.Reader[16];
- PackedInts.Reader[] startPointersDeltas = new PackedInts.Reader[16];
-
- final int packedIntsVersion = fieldsIndexIn.readVInt();
-
- int blockCount = 0;
-
- for (; ; ) {
- final int numChunks = fieldsIndexIn.readVInt();
- if (numChunks == 0) {
- break;
- }
- if (blockCount == docBases.length) {
- final int newSize = ArrayUtil.oversize(blockCount + 1, 8);
- docBases = ArrayUtil.growExact(docBases, newSize);
- startPointers = ArrayUtil.growExact(startPointers, newSize);
- avgChunkDocs = ArrayUtil.growExact(avgChunkDocs, newSize);
- avgChunkSizes = ArrayUtil.growExact(avgChunkSizes, newSize);
- docBasesDeltas = ArrayUtil.growExact(docBasesDeltas, newSize);
- startPointersDeltas = ArrayUtil.growExact(startPointersDeltas, newSize);
- }
-
- // doc bases
- docBases[blockCount] = fieldsIndexIn.readVInt();
- avgChunkDocs[blockCount] = fieldsIndexIn.readVInt();
- final int bitsPerDocBase = fieldsIndexIn.readVInt();
- if (bitsPerDocBase > 32) {
- throw new CorruptIndexException(
- "Corrupted bitsPerDocBase: " + bitsPerDocBase, fieldsIndexIn);
- }
- docBasesDeltas[blockCount] =
- LegacyPackedInts.getReaderNoHeader(
- fieldsIndexIn,
- PackedInts.Format.PACKED,
- packedIntsVersion,
- numChunks,
- bitsPerDocBase);
-
- // start pointers
- startPointers[blockCount] = fieldsIndexIn.readVLong();
- avgChunkSizes[blockCount] = fieldsIndexIn.readVLong();
- final int bitsPerStartPointer = fieldsIndexIn.readVInt();
- if (bitsPerStartPointer > 64) {
- throw new CorruptIndexException(
- "Corrupted bitsPerStartPointer: " + bitsPerStartPointer, fieldsIndexIn);
- }
- startPointersDeltas[blockCount] =
- LegacyPackedInts.getReaderNoHeader(
- fieldsIndexIn,
- PackedInts.Format.PACKED,
- packedIntsVersion,
- numChunks,
- bitsPerStartPointer);
-
- ++blockCount;
- }
-
- this.docBases = ArrayUtil.copyOfSubArray(docBases, 0, blockCount);
- this.startPointers = ArrayUtil.copyOfSubArray(startPointers, 0, blockCount);
- this.avgChunkDocs = ArrayUtil.copyOfSubArray(avgChunkDocs, 0, blockCount);
- this.avgChunkSizes = ArrayUtil.copyOfSubArray(avgChunkSizes, 0, blockCount);
- this.docBasesDeltas = ArrayUtil.copyOfSubArray(docBasesDeltas, 0, blockCount);
- this.startPointersDeltas = ArrayUtil.copyOfSubArray(startPointersDeltas, 0, blockCount);
- }
-
- private int block(int docID) {
- int lo = 0, hi = docBases.length - 1;
- while (lo <= hi) {
- final int mid = (lo + hi) >>> 1;
- final int midValue = docBases[mid];
- if (midValue == docID) {
- return mid;
- } else if (midValue < docID) {
- lo = mid + 1;
- } else {
- hi = mid - 1;
- }
- }
- return hi;
- }
-
- private int relativeDocBase(int block, int relativeChunk) {
- final int expected = avgChunkDocs[block] * relativeChunk;
- final long delta = zigZagDecode(docBasesDeltas[block].get(relativeChunk));
- return expected + (int) delta;
- }
-
- private long relativeStartPointer(int block, int relativeChunk) {
- final long expected = avgChunkSizes[block] * relativeChunk;
- final long delta = zigZagDecode(startPointersDeltas[block].get(relativeChunk));
- return expected + delta;
- }
-
- private int relativeChunk(int block, int relativeDoc) {
- int lo = 0, hi = docBasesDeltas[block].size() - 1;
- while (lo <= hi) {
- final int mid = (lo + hi) >>> 1;
- final int midValue = relativeDocBase(block, mid);
- if (midValue == relativeDoc) {
- return mid;
- } else if (midValue < relativeDoc) {
- lo = mid + 1;
- } else {
- hi = mid - 1;
- }
- }
- return hi;
- }
-
- @Override
- long getStartPointer(int docID) {
- if (docID < 0 || docID >= maxDoc) {
- throw new IllegalArgumentException("docID out of range [0-" + maxDoc + "]: " + docID);
- }
- final int block = block(docID);
- final int relativeChunk = relativeChunk(block, docID - docBases[block]);
- return startPointers[block] + relativeStartPointer(block, relativeChunk);
- }
-
- @Override
- public LegacyFieldsIndexReader clone() {
- return this;
- }
-
- @Override
- public String toString() {
- return getClass().getSimpleName() + "(blocks=" + docBases.length + ")";
- }
-
- @Override
- public void close() throws IOException {
- // nothing to do
- }
-
- @Override
- void checkIntegrity() throws IOException {
- // nothing to do, the index is checked at open time
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingStoredFieldsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingStoredFieldsFormat.java
deleted file mode 100644
index a1be277abc7..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingStoredFieldsFormat.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50.compressing;
-
-import java.io.IOException;
-import org.apache.lucene.backward_codecs.compressing.CompressionMode;
-import org.apache.lucene.backward_codecs.packed.LegacyDirectMonotonicWriter;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.StoredFieldsFormat;
-import org.apache.lucene.codecs.StoredFieldsReader;
-import org.apache.lucene.codecs.StoredFieldsWriter;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.MergePolicy;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-
-/**
- * A {@link StoredFieldsFormat} that compresses documents in chunks in order to improve the
- * compression ratio.
- *
- * <p>For a chunk size of <var>chunkSize</var> bytes, this {@link StoredFieldsFormat} does not
- * support documents larger than (<code>2<sup>31</sup> - chunkSize</code>) bytes.
- *
- * <p>For optimal performance, you should use a {@link MergePolicy} that returns segments that have
- * the biggest byte size first.
- *
- * @lucene.experimental
- */
-public class Lucene50CompressingStoredFieldsFormat extends StoredFieldsFormat {
-
- /** format name */
- protected final String formatName;
-
- /** segment suffix */
- protected final String segmentSuffix;
-
- /** compression mode */
- protected final CompressionMode compressionMode;
-
- /** chunk size */
- protected final int chunkSize;
-
- /** max docs per chunk */
- protected final int maxDocsPerChunk;
-
- /** block shift */
- protected final int blockShift;
-
- /**
- * Create a new {@link Lucene50CompressingStoredFieldsFormat} with an empty segment suffix.
- *
- * @see Lucene50CompressingStoredFieldsFormat#Lucene50CompressingStoredFieldsFormat(String,
- * String, CompressionMode, int, int, int)
- */
- public Lucene50CompressingStoredFieldsFormat(
- String formatName,
- CompressionMode compressionMode,
- int chunkSize,
- int maxDocsPerChunk,
- int blockShift) {
- this(formatName, "", compressionMode, chunkSize, maxDocsPerChunk, blockShift);
- }
-
- /**
- * Create a new {@link Lucene50CompressingStoredFieldsFormat}.
- *
- * <p><code>formatName</code> is the name of the format. This name will be used in the file
- * formats to perform {@link CodecUtil#checkIndexHeader codec header checks}.
- *
- * <p><code>segmentSuffix</code> is the segment suffix. This suffix is added to the result file
- * name only if it's not the empty string.
- *
- * <p>The <code>compressionMode</code> parameter allows you to choose between compression
- * algorithms that have various compression and decompression speeds so that you can pick the one
- * that best fits your indexing and searching throughput. You should never instantiate two {@link
- * Lucene50CompressingStoredFieldsFormat}s that have the same name but different {@link
- * CompressionMode}s.
- *
- * <p><code>chunkSize</code> is the minimum byte size of a chunk of documents. A value of <code>1
- * </code> can make sense if there is redundancy across fields. <code>maxDocsPerChunk</code> is an
- * upperbound on how many docs may be stored in a single chunk. This is to bound the cpu costs for
- * highly compressible data.
- *
- * <p>Higher values of <code>chunkSize</code> should improve the compression ratio but will
- * require more memory at indexing time and might make document loading a little slower (depending
- * on the size of your OS cache compared to the size of your index).
- *
- * @param formatName the name of the {@link StoredFieldsFormat}
- * @param compressionMode the {@link CompressionMode} to use
- * @param chunkSize the minimum number of bytes of a single chunk of stored documents
- * @param maxDocsPerChunk the maximum number of documents in a single chunk
- * @param blockShift the log in base 2 of number of chunks to store in an index block
- * @see CompressionMode
- */
- public Lucene50CompressingStoredFieldsFormat(
- String formatName,
- String segmentSuffix,
- CompressionMode compressionMode,
- int chunkSize,
- int maxDocsPerChunk,
- int blockShift) {
- this.formatName = formatName;
- this.segmentSuffix = segmentSuffix;
- this.compressionMode = compressionMode;
- if (chunkSize < 1) {
- throw new IllegalArgumentException("chunkSize must be >= 1");
- }
- this.chunkSize = chunkSize;
- if (maxDocsPerChunk < 1) {
- throw new IllegalArgumentException("maxDocsPerChunk must be >= 1");
- }
- this.maxDocsPerChunk = maxDocsPerChunk;
- if (blockShift < LegacyDirectMonotonicWriter.MIN_BLOCK_SHIFT
- || blockShift > LegacyDirectMonotonicWriter.MAX_BLOCK_SHIFT) {
- throw new IllegalArgumentException(
- "blockSize must be in "
- + LegacyDirectMonotonicWriter.MIN_BLOCK_SHIFT
- + "-"
- + LegacyDirectMonotonicWriter.MAX_BLOCK_SHIFT
- + ", got "
- + blockShift);
- }
- this.blockShift = blockShift;
- }
-
- @Override
- public StoredFieldsReader fieldsReader(
- Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
- return new Lucene50CompressingStoredFieldsReader(
- directory, si, segmentSuffix, fn, context, formatName, compressionMode);
- }
-
- @Override
- public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context)
- throws IOException {
- throw new UnsupportedOperationException("Old formats can't be used for writing");
- }
-
- @Override
- public String toString() {
- return getClass().getSimpleName()
- + "(compressionMode="
- + compressionMode
- + ", chunkSize="
- + chunkSize
- + ", maxDocsPerChunk="
- + maxDocsPerChunk
- + ", blockShift="
- + blockShift
- + ")";
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingStoredFieldsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingStoredFieldsReader.java
deleted file mode 100644
index edab3b0d8c2..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingStoredFieldsReader.java
+++ /dev/null
@@ -1,765 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50.compressing;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.util.Arrays;
-import org.apache.lucene.backward_codecs.compressing.CompressionMode;
-import org.apache.lucene.backward_codecs.compressing.Decompressor;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.StoredFieldsReader;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.StoredFieldDataInput;
-import org.apache.lucene.index.StoredFieldVisitor;
-import org.apache.lucene.store.AlreadyClosedException;
-import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BitUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.LongsRef;
-import org.apache.lucene.util.packed.PackedInts;
-
-/**
- * {@link StoredFieldsReader} impl for {@link Lucene50CompressingStoredFieldsFormat}.
- *
- * @lucene.experimental
- */
-public final class Lucene50CompressingStoredFieldsReader extends StoredFieldsReader {
-
- /** Extension of stored fields file */
- public static final String FIELDS_EXTENSION = "fdt";
-
- /** Extension of stored fields index */
- public static final String INDEX_EXTENSION = "fdx";
-
- /** Extension of stored fields meta */
- public static final String META_EXTENSION = "fdm";
-
- /** Codec name for the index. */
- public static final String INDEX_CODEC_NAME = "Lucene85FieldsIndex";
-
- static final int STRING = 0x00;
- static final int BYTE_ARR = 0x01;
- static final int NUMERIC_INT = 0x02;
- static final int NUMERIC_FLOAT = 0x03;
- static final int NUMERIC_LONG = 0x04;
- static final int NUMERIC_DOUBLE = 0x05;
-
- static final int TYPE_BITS = PackedInts.bitsRequired(NUMERIC_DOUBLE);
- static final int TYPE_MASK = (int) PackedInts.maxValue(TYPE_BITS);
-
- static final int VERSION_START = 1;
- static final int VERSION_OFFHEAP_INDEX = 2;
-
- /** Version where all metadata were moved to the meta file. */
- static final int VERSION_META = 3;
-
- /**
- * Version where numChunks is explicitly recorded in meta file and a dirty chunk bit is recorded
- * in each chunk
- */
- static final int VERSION_NUM_CHUNKS = 4;
-
- static final int VERSION_CURRENT = VERSION_NUM_CHUNKS;
- static final int META_VERSION_START = 0;
-
- // for compression of timestamps
- static final long SECOND = 1000L;
- static final long HOUR = 60 * 60 * SECOND;
- static final long DAY = 24 * HOUR;
- static final int SECOND_ENCODING = 0x40;
- static final int HOUR_ENCODING = 0x80;
- static final int DAY_ENCODING = 0xC0;
-
- private final int version;
- private final FieldInfos fieldInfos;
- private final FieldsIndex indexReader;
- private final long maxPointer;
- private final IndexInput fieldsStream;
- private final int chunkSize;
- private final int packedIntsVersion;
- private final CompressionMode compressionMode;
- private final Decompressor decompressor;
- private final int numDocs;
- private final boolean merging;
- private final BlockState state;
- private boolean closed;
-
- // used by clone
- private Lucene50CompressingStoredFieldsReader(
- Lucene50CompressingStoredFieldsReader reader, boolean merging) {
- this.version = reader.version;
- this.fieldInfos = reader.fieldInfos;
- this.fieldsStream = reader.fieldsStream.clone();
- this.indexReader = reader.indexReader.clone();
- this.maxPointer = reader.maxPointer;
- this.chunkSize = reader.chunkSize;
- this.packedIntsVersion = reader.packedIntsVersion;
- this.compressionMode = reader.compressionMode;
- this.decompressor = reader.decompressor.clone();
- this.numDocs = reader.numDocs;
- this.merging = merging;
- this.state = new BlockState();
- this.closed = false;
- }
-
- /** Sole constructor. */
- public Lucene50CompressingStoredFieldsReader(
- Directory d,
- SegmentInfo si,
- String segmentSuffix,
- FieldInfos fn,
- IOContext context,
- String formatName,
- CompressionMode compressionMode)
- throws IOException {
- this.compressionMode = compressionMode;
- final String segment = si.name;
- boolean success = false;
- fieldInfos = fn;
- numDocs = si.maxDoc();
-
- final String fieldsStreamFN =
- IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION);
- ChecksumIndexInput metaIn = null;
- try {
- // Open the data file
- fieldsStream = EndiannessReverserUtil.openInput(d, fieldsStreamFN, context);
- version =
- CodecUtil.checkIndexHeader(
- fieldsStream, formatName, VERSION_START, VERSION_CURRENT, si.getId(), segmentSuffix);
- assert CodecUtil.indexHeaderLength(formatName, segmentSuffix)
- == fieldsStream.getFilePointer();
-
- if (version >= VERSION_OFFHEAP_INDEX) {
- final String metaStreamFN =
- IndexFileNames.segmentFileName(segment, segmentSuffix, META_EXTENSION);
- metaIn = EndiannessReverserUtil.openChecksumInput(d, metaStreamFN, IOContext.READONCE);
- CodecUtil.checkIndexHeader(
- metaIn,
- INDEX_CODEC_NAME + "Meta",
- META_VERSION_START,
- version,
- si.getId(),
- segmentSuffix);
- }
- if (version >= VERSION_META) {
- chunkSize = metaIn.readVInt();
- packedIntsVersion = metaIn.readVInt();
- } else {
- chunkSize = fieldsStream.readVInt();
- packedIntsVersion = fieldsStream.readVInt();
- }
-
- decompressor = compressionMode.newDecompressor();
- this.merging = false;
- this.state = new BlockState();
-
- // NOTE: data file is too costly to verify checksum against all the bytes on open,
- // but for now we at least verify proper structure of the checksum footer: which looks
- // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
- // such as file truncation.
- CodecUtil.retrieveChecksum(fieldsStream);
-
- long maxPointer = -1;
- FieldsIndex indexReader = null;
-
- if (version < VERSION_OFFHEAP_INDEX) {
- // Load the index into memory
- final String indexName = IndexFileNames.segmentFileName(segment, segmentSuffix, "fdx");
- try (ChecksumIndexInput indexStream =
- EndiannessReverserUtil.openChecksumInput(d, indexName, context)) {
- Throwable priorE = null;
- try {
- assert formatName.endsWith("Data");
- final String codecNameIdx =
- formatName.substring(0, formatName.length() - "Data".length()) + "Index";
- final int version2 =
- CodecUtil.checkIndexHeader(
- indexStream,
- codecNameIdx,
- VERSION_START,
- VERSION_CURRENT,
- si.getId(),
- segmentSuffix);
- if (version != version2) {
- throw new CorruptIndexException(
- "Version mismatch between stored fields index and data: "
- + version2
- + " != "
- + version,
- indexStream);
- }
- assert CodecUtil.indexHeaderLength(codecNameIdx, segmentSuffix)
- == indexStream.getFilePointer();
- indexReader = new LegacyFieldsIndexReader(indexStream, si);
- maxPointer = indexStream.readVLong();
- } catch (Throwable exception) {
- priorE = exception;
- } finally {
- CodecUtil.checkFooter(indexStream, priorE);
- }
- }
- } else {
- FieldsIndexReader fieldsIndexReader =
- new FieldsIndexReader(
- d, si.name, segmentSuffix, INDEX_EXTENSION, INDEX_CODEC_NAME, si.getId(), metaIn);
- indexReader = fieldsIndexReader;
- maxPointer = fieldsIndexReader.getMaxPointer();
- }
-
- this.maxPointer = maxPointer;
- this.indexReader = indexReader;
-
- if (version >= VERSION_NUM_CHUNKS) {
- // discard num_chunks
- metaIn.readVLong();
- }
- if (version >= VERSION_META) {
- // consume dirty chunks/docs stats we wrote
- metaIn.readVLong();
- metaIn.readVLong();
- }
-
- if (metaIn != null) {
- CodecUtil.checkFooter(metaIn, null);
- metaIn.close();
- }
-
- success = true;
- } catch (Throwable t) {
- if (metaIn != null) {
- CodecUtil.checkFooter(metaIn, t);
- throw new AssertionError("unreachable");
- } else {
- throw t;
- }
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(this, metaIn);
- }
- }
- }
-
- /**
- * @throws AlreadyClosedException if this FieldsReader is closed
- */
- private void ensureOpen() throws AlreadyClosedException {
- if (closed) {
- throw new AlreadyClosedException("this FieldsReader is closed");
- }
- }
-
- /** Close the underlying {@link IndexInput}s. */
- @Override
- public void close() throws IOException {
- if (!closed) {
- IOUtils.close(indexReader, fieldsStream);
- closed = true;
- }
- }
-
- private static void readField(DataInput in, StoredFieldVisitor visitor, FieldInfo info, int bits)
- throws IOException {
- switch (bits & TYPE_MASK) {
- case BYTE_ARR:
- int length = in.readVInt();
- visitor.binaryField(info, new StoredFieldDataInput(in, length));
- break;
- case STRING:
- visitor.stringField(info, in.readString());
- break;
- case NUMERIC_INT:
- visitor.intField(info, in.readZInt());
- break;
- case NUMERIC_FLOAT:
- visitor.floatField(info, readZFloat(in));
- break;
- case NUMERIC_LONG:
- visitor.longField(info, readTLong(in));
- break;
- case NUMERIC_DOUBLE:
- visitor.doubleField(info, readZDouble(in));
- break;
- default:
- throw new AssertionError("Unknown type flag: " + Integer.toHexString(bits));
- }
- }
-
- private static void skipField(DataInput in, int bits) throws IOException {
- switch (bits & TYPE_MASK) {
- case BYTE_ARR:
- case STRING:
- final int length = in.readVInt();
- in.skipBytes(length);
- break;
- case NUMERIC_INT:
- in.readZInt();
- break;
- case NUMERIC_FLOAT:
- readZFloat(in);
- break;
- case NUMERIC_LONG:
- readTLong(in);
- break;
- case NUMERIC_DOUBLE:
- readZDouble(in);
- break;
- default:
- throw new AssertionError("Unknown type flag: " + Integer.toHexString(bits));
- }
- }
-
- /**
- * Reads a float in a variable-length format. Reads between one and five bytes. Small integral
- * values typically take fewer bytes.
- */
- static float readZFloat(DataInput in) throws IOException {
- int b = in.readByte() & 0xFF;
- if (b == 0xFF) {
- // negative value
- return Float.intBitsToFloat(in.readInt());
- } else if ((b & 0x80) != 0) {
- // small integer [-1..125]
- return (b & 0x7f) - 1;
- } else {
- // positive float
- int bits = b << 24 | ((in.readShort() & 0xFFFF) << 8) | (in.readByte() & 0xFF);
- return Float.intBitsToFloat(bits);
- }
- }
-
- /**
- * Reads a double in a variable-length format. Reads between one and nine bytes. Small integral
- * values typically take fewer bytes.
- */
- static double readZDouble(DataInput in) throws IOException {
- int b = in.readByte() & 0xFF;
- if (b == 0xFF) {
- // negative value
- return Double.longBitsToDouble(in.readLong());
- } else if (b == 0xFE) {
- // float
- return Float.intBitsToFloat(in.readInt());
- } else if ((b & 0x80) != 0) {
- // small integer [-1..124]
- return (b & 0x7f) - 1;
- } else {
- // positive double
- long bits =
- ((long) b) << 56
- | ((in.readInt() & 0xFFFFFFFFL) << 24)
- | ((in.readShort() & 0xFFFFL) << 8)
- | (in.readByte() & 0xFFL);
- return Double.longBitsToDouble(bits);
- }
- }
-
- /**
- * Reads a long in a variable-length format. Reads between one andCorePropLo nine bytes. Small
- * values typically take fewer bytes.
- */
- static long readTLong(DataInput in) throws IOException {
- int header = in.readByte() & 0xFF;
-
- long bits = header & 0x1F;
- if ((header & 0x20) != 0) {
- // continuation bit
- bits |= in.readVLong() << 5;
- }
-
- long l = BitUtil.zigZagDecode(bits);
-
- switch (header & DAY_ENCODING) {
- case SECOND_ENCODING:
- l *= SECOND;
- break;
- case HOUR_ENCODING:
- l *= HOUR;
- break;
- case DAY_ENCODING:
- l *= DAY;
- break;
- case 0:
- // uncompressed
- break;
- default:
- throw new AssertionError();
- }
-
- return l;
- }
-
- /**
- * A serialized document, you need to decode its input in order to get an actual {@link Document}.
- */
- static class SerializedDocument {
-
- // the serialized data
- final DataInput in;
-
- // the number of bytes on which the document is encoded
- final int length;
-
- // the number of stored fields
- final int numStoredFields;
-
- private SerializedDocument(DataInput in, int length, int numStoredFields) {
- this.in = in;
- this.length = length;
- this.numStoredFields = numStoredFields;
- }
- }
-
- /** Keeps state about the current block of documents. */
- private class BlockState {
-
- private int docBase, chunkDocs;
-
- // whether the block has been sliced, this happens for large documents
- private boolean sliced;
-
- private long[] offsets = LongsRef.EMPTY_LONGS;
- private long[] numStoredFields = LongsRef.EMPTY_LONGS;
-
- // the start pointer at which you can read the compressed documents
- private long startPointer;
-
- private final BytesRef spare;
- private final BytesRef bytes;
-
- BlockState() {
- if (merging) {
- spare = new BytesRef();
- bytes = new BytesRef();
- } else {
- spare = bytes = null;
- }
- }
-
- boolean contains(int docID) {
- return docID >= docBase && docID < docBase + chunkDocs;
- }
-
- /** Reset this block so that it stores state for the block that contains the given doc id. */
- void reset(int docID) throws IOException {
- boolean success = false;
- try {
- doReset(docID);
- success = true;
- } finally {
- if (success == false) {
- // if the read failed, set chunkDocs to 0 so that it does not
- // contain any docs anymore and is not reused. This should help
- // get consistent exceptions when trying to get several
- // documents which are in the same corrupted block since it will
- // force the header to be decoded again
- chunkDocs = 0;
- }
- }
- }
-
- private void doReset(int docID) throws IOException {
- docBase = fieldsStream.readVInt();
- final int token = fieldsStream.readVInt();
- chunkDocs = version >= VERSION_NUM_CHUNKS ? token >>> 2 : token >>> 1;
- if (contains(docID) == false || docBase + chunkDocs > numDocs) {
- throw new CorruptIndexException(
- "Corrupted: docID="
- + docID
- + ", docBase="
- + docBase
- + ", chunkDocs="
- + chunkDocs
- + ", numDocs="
- + numDocs,
- fieldsStream);
- }
-
- sliced = (token & 1) != 0;
-
- offsets = ArrayUtil.grow(offsets, chunkDocs + 1);
- numStoredFields = ArrayUtil.grow(numStoredFields, chunkDocs);
-
- if (chunkDocs == 1) {
- numStoredFields[0] = fieldsStream.readVInt();
- offsets[1] = fieldsStream.readVInt();
- } else {
- // Number of stored fields per document
- final int bitsPerStoredFields = fieldsStream.readVInt();
- if (bitsPerStoredFields == 0) {
- Arrays.fill(numStoredFields, 0, chunkDocs, fieldsStream.readVInt());
- } else if (bitsPerStoredFields > 31) {
- throw new CorruptIndexException(
- "bitsPerStoredFields=" + bitsPerStoredFields, fieldsStream);
- } else {
- final PackedInts.ReaderIterator it =
- PackedInts.getReaderIteratorNoHeader(
- fieldsStream,
- PackedInts.Format.PACKED,
- packedIntsVersion,
- chunkDocs,
- bitsPerStoredFields,
- 1024);
- for (int i = 0; i < chunkDocs; ) {
- final LongsRef next = it.next(Integer.MAX_VALUE);
- System.arraycopy(next.longs, next.offset, numStoredFields, i, next.length);
- i += next.length;
- }
- }
-
- // The stream encodes the length of each document and we decode
- // it into a list of monotonically increasing offsets
- final int bitsPerLength = fieldsStream.readVInt();
- if (bitsPerLength == 0) {
- final int length = fieldsStream.readVInt();
- for (int i = 0; i < chunkDocs; ++i) {
- offsets[1 + i] = (1 + i) * (long) length;
- }
- } else if (bitsPerStoredFields > 31) {
- throw new CorruptIndexException("bitsPerLength=" + bitsPerLength, fieldsStream);
- } else {
- final PackedInts.ReaderIterator it =
- PackedInts.getReaderIteratorNoHeader(
- fieldsStream,
- PackedInts.Format.PACKED,
- packedIntsVersion,
- chunkDocs,
- bitsPerLength,
- 1024);
- for (int i = 0; i < chunkDocs; ) {
- final LongsRef next = it.next(Integer.MAX_VALUE);
- System.arraycopy(next.longs, next.offset, offsets, i + 1, next.length);
- i += next.length;
- }
- for (int i = 0; i < chunkDocs; ++i) {
- offsets[i + 1] += offsets[i];
- }
- }
-
- // Additional validation: only the empty document has a serialized length of 0
- for (int i = 0; i < chunkDocs; ++i) {
- final long len = offsets[i + 1] - offsets[i];
- final long storedFields = numStoredFields[i];
- if ((len == 0) != (storedFields == 0)) {
- throw new CorruptIndexException(
- "length=" + len + ", numStoredFields=" + storedFields, fieldsStream);
- }
- }
- }
-
- startPointer = fieldsStream.getFilePointer();
-
- if (merging) {
- final int totalLength = Math.toIntExact(offsets[chunkDocs]);
- // decompress eagerly
- if (sliced) {
- bytes.offset = bytes.length = 0;
- for (int decompressed = 0; decompressed < totalLength; ) {
- final int toDecompress = Math.min(totalLength - decompressed, chunkSize);
- decompressor.decompress(fieldsStream, toDecompress, 0, toDecompress, spare);
- bytes.bytes = ArrayUtil.grow(bytes.bytes, bytes.length + spare.length);
- System.arraycopy(spare.bytes, spare.offset, bytes.bytes, bytes.length, spare.length);
- bytes.length += spare.length;
- decompressed += toDecompress;
- }
- } else {
- decompressor.decompress(fieldsStream, totalLength, 0, totalLength, bytes);
- }
- if (bytes.length != totalLength) {
- throw new CorruptIndexException(
- "Corrupted: expected chunk size = " + totalLength + ", got " + bytes.length,
- fieldsStream);
- }
- }
- }
-
- /**
- * Get the serialized representation of the given docID. This docID has to be contained in the
- * current block.
- */
- SerializedDocument document(int docID) throws IOException {
- if (contains(docID) == false) {
- throw new IllegalArgumentException();
- }
-
- final int index = docID - docBase;
- final int offset = Math.toIntExact(offsets[index]);
- final int length = Math.toIntExact(offsets[index + 1]) - offset;
- final int totalLength = Math.toIntExact(offsets[chunkDocs]);
- final int numStoredFields = Math.toIntExact(this.numStoredFields[index]);
-
- final BytesRef bytes;
- if (merging) {
- bytes = this.bytes;
- } else {
- bytes = new BytesRef();
- }
-
- final DataInput documentInput;
- if (length == 0) {
- // empty
- documentInput = new ByteArrayDataInput();
- } else if (merging) {
- // already decompressed
- documentInput = new ByteArrayDataInput(bytes.bytes, bytes.offset + offset, length);
- } else if (sliced) {
- fieldsStream.seek(startPointer);
- decompressor.decompress(
- fieldsStream, chunkSize, offset, Math.min(length, chunkSize - offset), bytes);
- documentInput =
- new DataInput() {
-
- int decompressed = bytes.length;
-
- void fillBuffer() throws IOException {
- assert decompressed <= length;
- if (decompressed == length) {
- throw new EOFException();
- }
- final int toDecompress = Math.min(length - decompressed, chunkSize);
- decompressor.decompress(fieldsStream, toDecompress, 0, toDecompress, bytes);
- decompressed += toDecompress;
- }
-
- @Override
- public byte readByte() throws IOException {
- if (bytes.length == 0) {
- fillBuffer();
- }
- --bytes.length;
- return bytes.bytes[bytes.offset++];
- }
-
- @Override
- public void readBytes(byte[] b, int offset, int len) throws IOException {
- while (len > bytes.length) {
- System.arraycopy(bytes.bytes, bytes.offset, b, offset, bytes.length);
- len -= bytes.length;
- offset += bytes.length;
- fillBuffer();
- }
- System.arraycopy(bytes.bytes, bytes.offset, b, offset, len);
- bytes.offset += len;
- bytes.length -= len;
- }
-
- @Override
- public void skipBytes(long numBytes) throws IOException {
- if (numBytes < 0) {
- throw new IllegalArgumentException("numBytes must be >= 0, got " + numBytes);
- }
- while (numBytes > bytes.length) {
- numBytes -= bytes.length;
- fillBuffer();
- }
- bytes.offset += numBytes;
- bytes.length -= numBytes;
- }
- };
- } else {
- fieldsStream.seek(startPointer);
- decompressor.decompress(fieldsStream, totalLength, offset, length, bytes);
- assert bytes.length == length;
- documentInput = new ByteArrayDataInput(bytes.bytes, bytes.offset, bytes.length);
- }
-
- return new SerializedDocument(
- EndiannessReverserUtil.wrapDataInput(documentInput), length, numStoredFields);
- }
- }
-
- SerializedDocument serializedDocument(int docID) throws IOException {
- if (state.contains(docID) == false) {
- fieldsStream.seek(indexReader.getStartPointer(docID));
- state.reset(docID);
- }
- assert state.contains(docID);
- return state.document(docID);
- }
-
- @Override
- public void document(int docID, StoredFieldVisitor visitor) throws IOException {
-
- final SerializedDocument doc = serializedDocument(docID);
-
- for (int fieldIDX = 0; fieldIDX < doc.numStoredFields; fieldIDX++) {
- final long infoAndBits = doc.in.readVLong();
- final int fieldNumber = (int) (infoAndBits >>> TYPE_BITS);
- final FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber);
-
- final int bits = (int) (infoAndBits & TYPE_MASK);
- assert bits <= NUMERIC_DOUBLE : "bits=" + Integer.toHexString(bits);
-
- switch (visitor.needsField(fieldInfo)) {
- case YES:
- readField(doc.in, visitor, fieldInfo, bits);
- break;
- case NO:
- if (fieldIDX
- == doc.numStoredFields - 1) { // don't skipField on last field value; treat like STOP
- return;
- }
- skipField(doc.in, bits);
- break;
- case STOP:
- return;
- }
- }
- }
-
- @Override
- public StoredFieldsReader clone() {
- ensureOpen();
- return new Lucene50CompressingStoredFieldsReader(this, false);
- }
-
- @Override
- public StoredFieldsReader getMergeInstance() {
- ensureOpen();
- return new Lucene50CompressingStoredFieldsReader(this, true);
- }
-
- @Override
- public void checkIntegrity() throws IOException {
- indexReader.checkIntegrity();
- CodecUtil.checksumEntireFile(fieldsStream);
- }
-
- @Override
- public String toString() {
- return getClass().getSimpleName()
- + "(mode="
- + compressionMode
- + ",chunksize="
- + chunkSize
- + ")";
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsFormat.java
deleted file mode 100644
index 7abf5772577..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsFormat.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50.compressing;
-
-import java.io.IOException;
-import org.apache.lucene.backward_codecs.compressing.CompressionMode;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.StoredFieldsFormat;
-import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.codecs.TermVectorsReader;
-import org.apache.lucene.codecs.TermVectorsWriter;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-
-/**
- * A {@link TermVectorsFormat} that compresses chunks of documents together in order to improve the
- * compression ratio.
- *
- * @lucene.experimental
- */
-public class Lucene50CompressingTermVectorsFormat extends TermVectorsFormat {
-
- /** format name */
- protected final String formatName;
-
- /** segment suffix */
- protected final String segmentSuffix;
-
- /** compression mode */
- protected final CompressionMode compressionMode;
-
- /** chunk size */
- protected final int chunkSize;
-
- /** block size */
- protected final int blockSize;
-
- /** max docs per chunk */
- protected final int maxDocsPerChunk;
-
- /**
- * Create a new {@link Lucene50CompressingTermVectorsFormat}.
- *
- * <p><code>formatName</code> is the name of the format. This name will be used in the file
- * formats to perform {@link CodecUtil#checkIndexHeader codec header checks}.
- *
- * <p>The <code>compressionMode</code> parameter allows you to choose between compression
- * algorithms that have various compression and decompression speeds so that you can pick the one
- * that best fits your indexing and searching throughput. You should never instantiate two {@link
- * Lucene50CompressingTermVectorsFormat}s that have the same name but different {@link
- * CompressionMode}s.
- *
- * <p><code>chunkSize</code> is the minimum byte size of a chunk of documents. Higher values of
- * <code>chunkSize</code> should improve the compression ratio but will require more memory at
- * indexing time and might make document loading a little slower (depending on the size of your OS
- * cache compared to the size of your index).
- *
- * @param formatName the name of the {@link StoredFieldsFormat}
- * @param segmentSuffix a suffix to append to files created by this format
- * @param compressionMode the {@link CompressionMode} to use
- * @param chunkSize the minimum number of bytes of a single chunk of stored documents
- * @param maxDocsPerChunk the maximum number of documents in a single chunk
- * @param blockSize the number of chunks to store in an index block.
- * @see CompressionMode
- */
- public Lucene50CompressingTermVectorsFormat(
- String formatName,
- String segmentSuffix,
- CompressionMode compressionMode,
- int chunkSize,
- int maxDocsPerChunk,
- int blockSize) {
- this.formatName = formatName;
- this.segmentSuffix = segmentSuffix;
- this.compressionMode = compressionMode;
- if (chunkSize < 1) {
- throw new IllegalArgumentException("chunkSize must be >= 1");
- }
- this.chunkSize = chunkSize;
- this.maxDocsPerChunk = maxDocsPerChunk;
- if (blockSize < 1) {
- throw new IllegalArgumentException("blockSize must be >= 1");
- }
- this.blockSize = blockSize;
- }
-
- @Override
- public final TermVectorsReader vectorsReader(
- Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context)
- throws IOException {
- return new Lucene50CompressingTermVectorsReader(
- directory, segmentInfo, segmentSuffix, fieldInfos, context, formatName, compressionMode);
- }
-
- @Override
- public TermVectorsWriter vectorsWriter(
- Directory directory, SegmentInfo segmentInfo, IOContext context) throws IOException {
- throw new UnsupportedOperationException("Old formats can't be used for writing");
- }
-
- @Override
- public String toString() {
- return getClass().getSimpleName()
- + "(compressionMode="
- + compressionMode
- + ", chunkSize="
- + chunkSize
- + ", maxDocsPerChunk="
- + maxDocsPerChunk
- + ", blockSize="
- + blockSize
- + ")";
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsReader.java
deleted file mode 100644
index bee4ea5e308..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsReader.java
+++ /dev/null
@@ -1,1303 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50.compressing;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-import org.apache.lucene.backward_codecs.compressing.CompressionMode;
-import org.apache.lucene.backward_codecs.compressing.Decompressor;
-import org.apache.lucene.backward_codecs.packed.LegacyPackedInts;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.TermVectorsReader;
-import org.apache.lucene.index.BaseTermsEnum;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.Fields;
-import org.apache.lucene.index.ImpactsEnum;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.SlowImpactsEnum;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.store.AlreadyClosedException;
-import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.LongsRef;
-import org.apache.lucene.util.packed.BlockPackedReaderIterator;
-import org.apache.lucene.util.packed.PackedInts;
-
-/**
- * {@link TermVectorsReader} for {@link Lucene50CompressingTermVectorsFormat}.
- *
- * @lucene.experimental
- */
-public final class Lucene50CompressingTermVectorsReader extends TermVectorsReader {
-
- static final String VECTORS_EXTENSION = "tvd";
- static final String VECTORS_INDEX_EXTENSION = "tvx";
- static final String VECTORS_META_EXTENSION = "tvm";
- static final String VECTORS_INDEX_CODEC_NAME = "Lucene85TermVectorsIndex";
-
- static final int VERSION_START = 1;
- static final int VERSION_OFFHEAP_INDEX = 2;
-
- /** Version where all metadata were moved to the meta file. */
- static final int VERSION_META = 3;
-
- /** Version where numChunks is explicitly recorded in meta file */
- static final int VERSION_NUM_CHUNKS = 4;
-
- static final int VERSION_CURRENT = VERSION_NUM_CHUNKS;
- static final int META_VERSION_START = 0;
-
- static final int PACKED_BLOCK_SIZE = 64;
-
- static final int POSITIONS = 0x01;
- static final int OFFSETS = 0x02;
- static final int PAYLOADS = 0x04;
- static final int FLAGS_BITS = PackedInts.bitsRequired(POSITIONS | OFFSETS | PAYLOADS);
-
- private final FieldInfos fieldInfos;
- final FieldsIndex indexReader;
- final IndexInput vectorsStream;
- private final int version;
- private final int packedIntsVersion;
- private final CompressionMode compressionMode;
- private final Decompressor decompressor;
- private final int chunkSize;
- private final int numDocs;
- private boolean closed;
- private final BlockPackedReaderIterator reader;
- private final long maxPointer; // end of the data section
-
- // used by clone
- private Lucene50CompressingTermVectorsReader(Lucene50CompressingTermVectorsReader reader) {
- this.fieldInfos = reader.fieldInfos;
- this.vectorsStream = reader.vectorsStream.clone();
- this.indexReader = reader.indexReader.clone();
- this.packedIntsVersion = reader.packedIntsVersion;
- this.compressionMode = reader.compressionMode;
- this.decompressor = reader.decompressor.clone();
- this.chunkSize = reader.chunkSize;
- this.numDocs = reader.numDocs;
- this.reader =
- new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, PACKED_BLOCK_SIZE, 0);
- this.version = reader.version;
- this.maxPointer = reader.maxPointer;
- this.closed = false;
- }
-
- /** Sole constructor. */
- public Lucene50CompressingTermVectorsReader(
- Directory d,
- SegmentInfo si,
- String segmentSuffix,
- FieldInfos fn,
- IOContext context,
- String formatName,
- CompressionMode compressionMode)
- throws IOException {
- this.compressionMode = compressionMode;
- final String segment = si.name;
- boolean success = false;
- fieldInfos = fn;
- numDocs = si.maxDoc();
-
- ChecksumIndexInput metaIn = null;
- try {
- // Open the data file
- final String vectorsStreamFN =
- IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION);
- vectorsStream = EndiannessReverserUtil.openInput(d, vectorsStreamFN, context);
- version =
- CodecUtil.checkIndexHeader(
- vectorsStream, formatName, VERSION_START, VERSION_CURRENT, si.getId(), segmentSuffix);
- assert CodecUtil.indexHeaderLength(formatName, segmentSuffix)
- == vectorsStream.getFilePointer();
-
- if (version >= VERSION_OFFHEAP_INDEX) {
- final String metaStreamFN =
- IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_META_EXTENSION);
- metaIn = EndiannessReverserUtil.openChecksumInput(d, metaStreamFN, IOContext.READONCE);
- CodecUtil.checkIndexHeader(
- metaIn,
- VECTORS_INDEX_CODEC_NAME + "Meta",
- META_VERSION_START,
- version,
- si.getId(),
- segmentSuffix);
- }
-
- if (version >= VERSION_META) {
- packedIntsVersion = metaIn.readVInt();
- chunkSize = metaIn.readVInt();
- } else {
- packedIntsVersion = vectorsStream.readVInt();
- chunkSize = vectorsStream.readVInt();
- }
-
- // NOTE: data file is too costly to verify checksum against all the bytes on open,
- // but for now we at least verify proper structure of the checksum footer: which looks
- // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
- // such as file truncation.
- CodecUtil.retrieveChecksum(vectorsStream);
-
- FieldsIndex indexReader = null;
- long maxPointer = -1;
-
- if (version < VERSION_OFFHEAP_INDEX) {
- // Load the index into memory
- final String indexName = IndexFileNames.segmentFileName(segment, segmentSuffix, "tvx");
- try (ChecksumIndexInput indexStream =
- EndiannessReverserUtil.openChecksumInput(d, indexName, context)) {
- Throwable priorE = null;
- try {
- assert formatName.endsWith("Data");
- final String codecNameIdx =
- formatName.substring(0, formatName.length() - "Data".length()) + "Index";
- final int version2 =
- CodecUtil.checkIndexHeader(
- indexStream,
- codecNameIdx,
- VERSION_START,
- VERSION_CURRENT,
- si.getId(),
- segmentSuffix);
- if (version != version2) {
- throw new CorruptIndexException(
- "Version mismatch between stored fields index and data: "
- + version
- + " != "
- + version2,
- indexStream);
- }
- assert CodecUtil.indexHeaderLength(codecNameIdx, segmentSuffix)
- == indexStream.getFilePointer();
- indexReader = new LegacyFieldsIndexReader(indexStream, si);
- maxPointer = indexStream.readVLong(); // the end of the data section
- } catch (Throwable exception) {
- priorE = exception;
- } finally {
- CodecUtil.checkFooter(indexStream, priorE);
- }
- }
- } else {
- FieldsIndexReader fieldsIndexReader =
- new FieldsIndexReader(
- d,
- si.name,
- segmentSuffix,
- VECTORS_INDEX_EXTENSION,
- VECTORS_INDEX_CODEC_NAME,
- si.getId(),
- metaIn);
- indexReader = fieldsIndexReader;
- maxPointer = fieldsIndexReader.getMaxPointer();
- }
-
- this.indexReader = indexReader;
- this.maxPointer = maxPointer;
-
- if (version >= VERSION_NUM_CHUNKS) {
- // consume num_chunks
- metaIn.readVLong();
- }
- if (version >= VERSION_META) {
- // consume dirty chunks/docs stats we wrote
- metaIn.readVLong();
- metaIn.readVLong();
- }
-
- decompressor = compressionMode.newDecompressor();
- this.reader =
- new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, PACKED_BLOCK_SIZE, 0);
-
- if (metaIn != null) {
- CodecUtil.checkFooter(metaIn, null);
- metaIn.close();
- }
-
- success = true;
- } catch (Throwable t) {
- if (metaIn != null) {
- CodecUtil.checkFooter(metaIn, t);
- throw new AssertionError("unreachable");
- } else {
- throw t;
- }
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(this, metaIn);
- }
- }
- }
-
- /**
- * @throws AlreadyClosedException if this TermVectorsReader is closed
- */
- private void ensureOpen() throws AlreadyClosedException {
- if (closed) {
- throw new AlreadyClosedException("this FieldsReader is closed");
- }
- }
-
- @Override
- public void close() throws IOException {
- if (!closed) {
- IOUtils.close(indexReader, vectorsStream);
- closed = true;
- }
- }
-
- @Override
- public TermVectorsReader clone() {
- return new Lucene50CompressingTermVectorsReader(this);
- }
-
- @Override
- public Fields get(int doc) throws IOException {
- ensureOpen();
-
- // seek to the right place
- {
- final long startPointer = indexReader.getStartPointer(doc);
- vectorsStream.seek(startPointer);
- }
-
- // decode
- // - docBase: first doc ID of the chunk
- // - chunkDocs: number of docs of the chunk
- final int docBase = vectorsStream.readVInt();
- final int chunkDocs = vectorsStream.readVInt();
- if (doc < docBase || doc >= docBase + chunkDocs || docBase + chunkDocs > numDocs) {
- throw new CorruptIndexException(
- "docBase=" + docBase + ",chunkDocs=" + chunkDocs + ",doc=" + doc, vectorsStream);
- }
-
- final int skip; // number of fields to skip
- final int numFields; // number of fields of the document we're looking for
- final int totalFields; // total number of fields of the chunk (sum for all docs)
- if (chunkDocs == 1) {
- skip = 0;
- numFields = totalFields = vectorsStream.readVInt();
- } else {
- reader.reset(vectorsStream, chunkDocs);
- int sum = 0;
- for (int i = docBase; i < doc; ++i) {
- sum += reader.next();
- }
- skip = sum;
- numFields = (int) reader.next();
- sum += numFields;
- for (int i = doc + 1; i < docBase + chunkDocs; ++i) {
- sum += reader.next();
- }
- totalFields = sum;
- }
-
- if (numFields == 0) {
- // no vectors
- return null;
- }
-
- // read field numbers that have term vectors
- final int[] fieldNums;
- {
- final int token = vectorsStream.readByte() & 0xFF;
- assert token != 0; // means no term vectors, cannot happen since we checked for numFields == 0
- final int bitsPerFieldNum = token & 0x1F;
- int totalDistinctFields = token >>> 5;
- if (totalDistinctFields == 0x07) {
- totalDistinctFields += vectorsStream.readVInt();
- }
- ++totalDistinctFields;
- final PackedInts.ReaderIterator it =
- PackedInts.getReaderIteratorNoHeader(
- vectorsStream,
- PackedInts.Format.PACKED,
- packedIntsVersion,
- totalDistinctFields,
- bitsPerFieldNum,
- 1);
- fieldNums = new int[totalDistinctFields];
- for (int i = 0; i < totalDistinctFields; ++i) {
- fieldNums[i] = (int) it.next();
- }
- }
-
- // read field numbers and flags
- final int[] fieldNumOffs = new int[numFields];
- final PackedInts.Reader flags;
- {
- final int bitsPerOff = PackedInts.bitsRequired(fieldNums.length - 1);
- final PackedInts.Reader allFieldNumOffs =
- LegacyPackedInts.getReaderNoHeader(
- vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, totalFields, bitsPerOff);
- switch (vectorsStream.readVInt()) {
- case 0:
- final PackedInts.Reader fieldFlags =
- LegacyPackedInts.getReaderNoHeader(
- vectorsStream,
- PackedInts.Format.PACKED,
- packedIntsVersion,
- fieldNums.length,
- FLAGS_BITS);
- PackedInts.Mutable f = PackedInts.getMutable(totalFields, FLAGS_BITS, PackedInts.COMPACT);
- for (int i = 0; i < totalFields; ++i) {
- final int fieldNumOff = (int) allFieldNumOffs.get(i);
- assert fieldNumOff >= 0 && fieldNumOff < fieldNums.length;
- final int fgs = (int) fieldFlags.get(fieldNumOff);
- f.set(i, fgs);
- }
- flags = f;
- break;
- case 1:
- flags =
- LegacyPackedInts.getReaderNoHeader(
- vectorsStream,
- PackedInts.Format.PACKED,
- packedIntsVersion,
- totalFields,
- FLAGS_BITS);
- break;
- default:
- throw new AssertionError();
- }
- for (int i = 0; i < numFields; ++i) {
- fieldNumOffs[i] = (int) allFieldNumOffs.get(skip + i);
- }
- }
-
- // number of terms per field for all fields
- final PackedInts.Reader numTerms;
- final int totalTerms;
- {
- final int bitsRequired = vectorsStream.readVInt();
- numTerms =
- LegacyPackedInts.getReaderNoHeader(
- vectorsStream,
- PackedInts.Format.PACKED,
- packedIntsVersion,
- totalFields,
- bitsRequired);
- int sum = 0;
- for (int i = 0; i < totalFields; ++i) {
- sum += numTerms.get(i);
- }
- totalTerms = sum;
- }
-
- // term lengths
- int docOff = 0, docLen = 0, totalLen;
- final int[] fieldLengths = new int[numFields];
- final int[][] prefixLengths = new int[numFields][];
- final int[][] suffixLengths = new int[numFields][];
- {
- reader.reset(vectorsStream, totalTerms);
- // skip
- int toSkip = 0;
- for (int i = 0; i < skip; ++i) {
- toSkip += numTerms.get(i);
- }
- reader.skip(toSkip);
- // read prefix lengths
- for (int i = 0; i < numFields; ++i) {
- final int termCount = (int) numTerms.get(skip + i);
- final int[] fieldPrefixLengths = new int[termCount];
- prefixLengths[i] = fieldPrefixLengths;
- for (int j = 0; j < termCount; ) {
- final LongsRef next = reader.next(termCount - j);
- for (int k = 0; k < next.length; ++k) {
- fieldPrefixLengths[j++] = (int) next.longs[next.offset + k];
- }
- }
- }
- reader.skip(totalTerms - reader.ord());
-
- reader.reset(vectorsStream, totalTerms);
- // skip
- toSkip = 0;
- for (int i = 0; i < skip; ++i) {
- for (int j = 0; j < numTerms.get(i); ++j) {
- docOff += reader.next();
- }
- }
- for (int i = 0; i < numFields; ++i) {
- final int termCount = (int) numTerms.get(skip + i);
- final int[] fieldSuffixLengths = new int[termCount];
- suffixLengths[i] = fieldSuffixLengths;
- for (int j = 0; j < termCount; ) {
- final LongsRef next = reader.next(termCount - j);
- for (int k = 0; k < next.length; ++k) {
- fieldSuffixLengths[j++] = (int) next.longs[next.offset + k];
- }
- }
- fieldLengths[i] = sum(suffixLengths[i]);
- docLen += fieldLengths[i];
- }
- totalLen = docOff + docLen;
- for (int i = skip + numFields; i < totalFields; ++i) {
- for (int j = 0; j < numTerms.get(i); ++j) {
- totalLen += reader.next();
- }
- }
- }
-
- // term freqs
- final int[] termFreqs = new int[totalTerms];
- {
- reader.reset(vectorsStream, totalTerms);
- for (int i = 0; i < totalTerms; ) {
- final LongsRef next = reader.next(totalTerms - i);
- for (int k = 0; k < next.length; ++k) {
- termFreqs[i++] = 1 + (int) next.longs[next.offset + k];
- }
- }
- }
-
- // total number of positions, offsets and payloads
- int totalPositions = 0, totalOffsets = 0, totalPayloads = 0;
- for (int i = 0, termIndex = 0; i < totalFields; ++i) {
- final int f = (int) flags.get(i);
- final int termCount = (int) numTerms.get(i);
- for (int j = 0; j < termCount; ++j) {
- final int freq = termFreqs[termIndex++];
- if ((f & POSITIONS) != 0) {
- totalPositions += freq;
- }
- if ((f & OFFSETS) != 0) {
- totalOffsets += freq;
- }
- if ((f & PAYLOADS) != 0) {
- totalPayloads += freq;
- }
- }
- assert i != totalFields - 1 || termIndex == totalTerms : termIndex + " " + totalTerms;
- }
-
- final int[][] positionIndex = positionIndex(skip, numFields, numTerms, termFreqs);
- final int[][] positions, startOffsets, lengths;
- if (totalPositions > 0) {
- positions =
- readPositions(
- skip,
- numFields,
- flags,
- numTerms,
- termFreqs,
- POSITIONS,
- totalPositions,
- positionIndex);
- } else {
- positions = new int[numFields][];
- }
-
- if (totalOffsets > 0) {
- // average number of chars per term
- final float[] charsPerTerm = new float[fieldNums.length];
- for (int i = 0; i < charsPerTerm.length; ++i) {
- charsPerTerm[i] = Float.intBitsToFloat(vectorsStream.readInt());
- }
- startOffsets =
- readPositions(
- skip, numFields, flags, numTerms, termFreqs, OFFSETS, totalOffsets, positionIndex);
- lengths =
- readPositions(
- skip, numFields, flags, numTerms, termFreqs, OFFSETS, totalOffsets, positionIndex);
-
- for (int i = 0; i < numFields; ++i) {
- final int[] fStartOffsets = startOffsets[i];
- final int[] fPositions = positions[i];
- // patch offsets from positions
- if (fStartOffsets != null && fPositions != null) {
- final float fieldCharsPerTerm = charsPerTerm[fieldNumOffs[i]];
- for (int j = 0; j < startOffsets[i].length; ++j) {
- fStartOffsets[j] += (int) (fieldCharsPerTerm * fPositions[j]);
- }
- }
- if (fStartOffsets != null) {
- final int[] fPrefixLengths = prefixLengths[i];
- final int[] fSuffixLengths = suffixLengths[i];
- final int[] fLengths = lengths[i];
- for (int j = 0, end = (int) numTerms.get(skip + i); j < end; ++j) {
- // delta-decode start offsets and patch lengths using term lengths
- final int termLength = fPrefixLengths[j] + fSuffixLengths[j];
- lengths[i][positionIndex[i][j]] += termLength;
- for (int k = positionIndex[i][j] + 1; k < positionIndex[i][j + 1]; ++k) {
- fStartOffsets[k] += fStartOffsets[k - 1];
- fLengths[k] += termLength;
- }
- }
- }
- }
- } else {
- startOffsets = lengths = new int[numFields][];
- }
- if (totalPositions > 0) {
- // delta-decode positions
- for (int i = 0; i < numFields; ++i) {
- final int[] fPositions = positions[i];
- final int[] fpositionIndex = positionIndex[i];
- if (fPositions != null) {
- for (int j = 0, end = (int) numTerms.get(skip + i); j < end; ++j) {
- // delta-decode start offsets
- for (int k = fpositionIndex[j] + 1; k < fpositionIndex[j + 1]; ++k) {
- fPositions[k] += fPositions[k - 1];
- }
- }
- }
- }
- }
-
- // payload lengths
- final int[][] payloadIndex = new int[numFields][];
- int totalPayloadLength = 0;
- int payloadOff = 0;
- int payloadLen = 0;
- if (totalPayloads > 0) {
- reader.reset(vectorsStream, totalPayloads);
- // skip
- int termIndex = 0;
- for (int i = 0; i < skip; ++i) {
- final int f = (int) flags.get(i);
- final int termCount = (int) numTerms.get(i);
- if ((f & PAYLOADS) != 0) {
- for (int j = 0; j < termCount; ++j) {
- final int freq = termFreqs[termIndex + j];
- for (int k = 0; k < freq; ++k) {
- final int l = (int) reader.next();
- payloadOff += l;
- }
- }
- }
- termIndex += termCount;
- }
- totalPayloadLength = payloadOff;
- // read doc payload lengths
- for (int i = 0; i < numFields; ++i) {
- final int f = (int) flags.get(skip + i);
- final int termCount = (int) numTerms.get(skip + i);
- if ((f & PAYLOADS) != 0) {
- final int totalFreq = positionIndex[i][termCount];
- payloadIndex[i] = new int[totalFreq + 1];
- int posIdx = 0;
- payloadIndex[i][posIdx] = payloadLen;
- for (int j = 0; j < termCount; ++j) {
- final int freq = termFreqs[termIndex + j];
- for (int k = 0; k < freq; ++k) {
- final int payloadLength = (int) reader.next();
- payloadLen += payloadLength;
- payloadIndex[i][posIdx + 1] = payloadLen;
- ++posIdx;
- }
- }
- assert posIdx == totalFreq;
- }
- termIndex += termCount;
- }
- totalPayloadLength += payloadLen;
- for (int i = skip + numFields; i < totalFields; ++i) {
- final int f = (int) flags.get(i);
- final int termCount = (int) numTerms.get(i);
- if ((f & PAYLOADS) != 0) {
- for (int j = 0; j < termCount; ++j) {
- final int freq = termFreqs[termIndex + j];
- for (int k = 0; k < freq; ++k) {
- totalPayloadLength += reader.next();
- }
- }
- }
- termIndex += termCount;
- }
- assert termIndex == totalTerms : termIndex + " " + totalTerms;
- }
-
- // decompress data
- final BytesRef suffixBytes = new BytesRef();
- decompressor.decompress(
- vectorsStream,
- totalLen + totalPayloadLength,
- docOff + payloadOff,
- docLen + payloadLen,
- suffixBytes);
- suffixBytes.length = docLen;
- final BytesRef payloadBytes =
- new BytesRef(suffixBytes.bytes, suffixBytes.offset + docLen, payloadLen);
-
- final int[] fieldFlags = new int[numFields];
- for (int i = 0; i < numFields; ++i) {
- fieldFlags[i] = (int) flags.get(skip + i);
- }
-
- final int[] fieldNumTerms = new int[numFields];
- for (int i = 0; i < numFields; ++i) {
- fieldNumTerms[i] = (int) numTerms.get(skip + i);
- }
-
- final int[][] fieldTermFreqs = new int[numFields][];
- {
- int termIdx = 0;
- for (int i = 0; i < skip; ++i) {
- termIdx += numTerms.get(i);
- }
- for (int i = 0; i < numFields; ++i) {
- final int termCount = (int) numTerms.get(skip + i);
- fieldTermFreqs[i] = new int[termCount];
- for (int j = 0; j < termCount; ++j) {
- fieldTermFreqs[i][j] = termFreqs[termIdx++];
- }
- }
- }
-
- assert sum(fieldLengths) == docLen : sum(fieldLengths) + " != " + docLen;
-
- return new TVFields(
- fieldNums,
- fieldFlags,
- fieldNumOffs,
- fieldNumTerms,
- fieldLengths,
- prefixLengths,
- suffixLengths,
- fieldTermFreqs,
- positionIndex,
- positions,
- startOffsets,
- lengths,
- payloadBytes,
- payloadIndex,
- suffixBytes);
- }
-
- // field -> term index -> position index
- private int[][] positionIndex(
- int skip, int numFields, PackedInts.Reader numTerms, int[] termFreqs) {
- final int[][] positionIndex = new int[numFields][];
- int termIndex = 0;
- for (int i = 0; i < skip; ++i) {
- final int termCount = (int) numTerms.get(i);
- termIndex += termCount;
- }
- for (int i = 0; i < numFields; ++i) {
- final int termCount = (int) numTerms.get(skip + i);
- positionIndex[i] = new int[termCount + 1];
- for (int j = 0; j < termCount; ++j) {
- final int freq = termFreqs[termIndex + j];
- positionIndex[i][j + 1] = positionIndex[i][j] + freq;
- }
- termIndex += termCount;
- }
- return positionIndex;
- }
-
- private int[][] readPositions(
- int skip,
- int numFields,
- PackedInts.Reader flags,
- PackedInts.Reader numTerms,
- int[] termFreqs,
- int flag,
- final int totalPositions,
- int[][] positionIndex)
- throws IOException {
- final int[][] positions = new int[numFields][];
- reader.reset(vectorsStream, totalPositions);
- // skip
- int toSkip = 0;
- int termIndex = 0;
- for (int i = 0; i < skip; ++i) {
- final int f = (int) flags.get(i);
- final int termCount = (int) numTerms.get(i);
- if ((f & flag) != 0) {
- for (int j = 0; j < termCount; ++j) {
- final int freq = termFreqs[termIndex + j];
- toSkip += freq;
- }
- }
- termIndex += termCount;
- }
- reader.skip(toSkip);
- // read doc positions
- for (int i = 0; i < numFields; ++i) {
- final int f = (int) flags.get(skip + i);
- final int termCount = (int) numTerms.get(skip + i);
- if ((f & flag) != 0) {
- final int totalFreq = positionIndex[i][termCount];
- final int[] fieldPositions = new int[totalFreq];
- positions[i] = fieldPositions;
- for (int j = 0; j < totalFreq; ) {
- final LongsRef nextPositions = reader.next(totalFreq - j);
- for (int k = 0; k < nextPositions.length; ++k) {
- fieldPositions[j++] = (int) nextPositions.longs[nextPositions.offset + k];
- }
- }
- }
- termIndex += termCount;
- }
- reader.skip(totalPositions - reader.ord());
- return positions;
- }
-
- private class TVFields extends Fields {
-
- private final int[] fieldNums, fieldFlags, fieldNumOffs, numTerms, fieldLengths;
- private final int[][] prefixLengths,
- suffixLengths,
- termFreqs,
- positionIndex,
- positions,
- startOffsets,
- lengths,
- payloadIndex;
- private final BytesRef suffixBytes, payloadBytes;
-
- public TVFields(
- int[] fieldNums,
- int[] fieldFlags,
- int[] fieldNumOffs,
- int[] numTerms,
- int[] fieldLengths,
- int[][] prefixLengths,
- int[][] suffixLengths,
- int[][] termFreqs,
- int[][] positionIndex,
- int[][] positions,
- int[][] startOffsets,
- int[][] lengths,
- BytesRef payloadBytes,
- int[][] payloadIndex,
- BytesRef suffixBytes) {
- this.fieldNums = fieldNums;
- this.fieldFlags = fieldFlags;
- this.fieldNumOffs = fieldNumOffs;
- this.numTerms = numTerms;
- this.fieldLengths = fieldLengths;
- this.prefixLengths = prefixLengths;
- this.suffixLengths = suffixLengths;
- this.termFreqs = termFreqs;
- this.positionIndex = positionIndex;
- this.positions = positions;
- this.startOffsets = startOffsets;
- this.lengths = lengths;
- this.payloadBytes = payloadBytes;
- this.payloadIndex = payloadIndex;
- this.suffixBytes = suffixBytes;
- }
-
- @Override
- public Iterator<String> iterator() {
- return new Iterator<>() {
- int i = 0;
-
- @Override
- public boolean hasNext() {
- return i < fieldNumOffs.length;
- }
-
- @Override
- public String next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- final int fieldNum = fieldNums[fieldNumOffs[i++]];
- return fieldInfos.fieldInfo(fieldNum).name;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
- }
-
- @Override
- public Terms terms(String field) throws IOException {
- final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
- if (fieldInfo == null) {
- return null;
- }
- int idx = -1;
- for (int i = 0; i < fieldNumOffs.length; ++i) {
- if (fieldNums[fieldNumOffs[i]] == fieldInfo.number) {
- idx = i;
- break;
- }
- }
-
- if (idx == -1 || numTerms[idx] == 0) {
- // no term
- return null;
- }
- int fieldOff = 0, fieldLen = -1;
- for (int i = 0; i < fieldNumOffs.length; ++i) {
- if (i < idx) {
- fieldOff += fieldLengths[i];
- } else {
- fieldLen = fieldLengths[i];
- break;
- }
- }
- assert fieldLen >= 0;
- return new TVTerms(
- numTerms[idx],
- fieldFlags[idx],
- prefixLengths[idx],
- suffixLengths[idx],
- termFreqs[idx],
- positionIndex[idx],
- positions[idx],
- startOffsets[idx],
- lengths[idx],
- payloadIndex[idx],
- payloadBytes,
- new BytesRef(suffixBytes.bytes, suffixBytes.offset + fieldOff, fieldLen));
- }
-
- @Override
- public int size() {
- return fieldNumOffs.length;
- }
- }
-
- private static class TVTerms extends Terms {
-
- private final int numTerms, flags;
- private final long totalTermFreq;
- private final int[] prefixLengths,
- suffixLengths,
- termFreqs,
- positionIndex,
- positions,
- startOffsets,
- lengths,
- payloadIndex;
- private final BytesRef termBytes, payloadBytes;
-
- TVTerms(
- int numTerms,
- int flags,
- int[] prefixLengths,
- int[] suffixLengths,
- int[] termFreqs,
- int[] positionIndex,
- int[] positions,
- int[] startOffsets,
- int[] lengths,
- int[] payloadIndex,
- BytesRef payloadBytes,
- BytesRef termBytes) {
- this.numTerms = numTerms;
- this.flags = flags;
- this.prefixLengths = prefixLengths;
- this.suffixLengths = suffixLengths;
- this.termFreqs = termFreqs;
- this.positionIndex = positionIndex;
- this.positions = positions;
- this.startOffsets = startOffsets;
- this.lengths = lengths;
- this.payloadIndex = payloadIndex;
- this.payloadBytes = payloadBytes;
- this.termBytes = termBytes;
- long ttf = 0;
- for (int tf : termFreqs) {
- ttf += tf;
- }
- this.totalTermFreq = ttf;
- }
-
- @Override
- public TermsEnum iterator() throws IOException {
- TVTermsEnum termsEnum = new TVTermsEnum();
- termsEnum.reset(
- numTerms,
- flags,
- prefixLengths,
- suffixLengths,
- termFreqs,
- positionIndex,
- positions,
- startOffsets,
- lengths,
- payloadIndex,
- payloadBytes,
- new ByteArrayDataInput(termBytes.bytes, termBytes.offset, termBytes.length));
- return termsEnum;
- }
-
- @Override
- public long size() throws IOException {
- return numTerms;
- }
-
- @Override
- public long getSumTotalTermFreq() throws IOException {
- return totalTermFreq;
- }
-
- @Override
- public long getSumDocFreq() throws IOException {
- return numTerms;
- }
-
- @Override
- public int getDocCount() throws IOException {
- return 1;
- }
-
- @Override
- public boolean hasFreqs() {
- return true;
- }
-
- @Override
- public boolean hasOffsets() {
- return (flags & OFFSETS) != 0;
- }
-
- @Override
- public boolean hasPositions() {
- return (flags & POSITIONS) != 0;
- }
-
- @Override
- public boolean hasPayloads() {
- return (flags & PAYLOADS) != 0;
- }
- }
-
- private static class TVTermsEnum extends BaseTermsEnum {
-
- private int numTerms, startPos, ord;
- private int[] prefixLengths,
- suffixLengths,
- termFreqs,
- positionIndex,
- positions,
- startOffsets,
- lengths,
- payloadIndex;
- private ByteArrayDataInput in;
- private BytesRef payloads;
- private final BytesRef term;
-
- private TVTermsEnum() {
- term = new BytesRef(16);
- }
-
- void reset(
- int numTerms,
- int flags,
- int[] prefixLengths,
- int[] suffixLengths,
- int[] termFreqs,
- int[] positionIndex,
- int[] positions,
- int[] startOffsets,
- int[] lengths,
- int[] payloadIndex,
- BytesRef payloads,
- ByteArrayDataInput in) {
- this.numTerms = numTerms;
- this.prefixLengths = prefixLengths;
- this.suffixLengths = suffixLengths;
- this.termFreqs = termFreqs;
- this.positionIndex = positionIndex;
- this.positions = positions;
- this.startOffsets = startOffsets;
- this.lengths = lengths;
- this.payloadIndex = payloadIndex;
- this.payloads = payloads;
- this.in = in;
- startPos = in.getPosition();
- reset();
- }
-
- void reset() {
- term.length = 0;
- in.setPosition(startPos);
- ord = -1;
- }
-
- @Override
- public BytesRef next() throws IOException {
- if (ord == numTerms - 1) {
- return null;
- } else {
- assert ord < numTerms;
- ++ord;
- }
-
- // read term
- term.offset = 0;
- term.length = prefixLengths[ord] + suffixLengths[ord];
- if (term.length > term.bytes.length) {
- term.bytes = ArrayUtil.grow(term.bytes, term.length);
- }
- in.readBytes(term.bytes, prefixLengths[ord], suffixLengths[ord]);
-
- return term;
- }
-
- @Override
- public SeekStatus seekCeil(BytesRef text) throws IOException {
- if (ord < numTerms && ord >= 0) {
- final int cmp = term().compareTo(text);
- if (cmp == 0) {
- return SeekStatus.FOUND;
- } else if (cmp > 0) {
- reset();
- }
- }
- // linear scan
- while (true) {
- final BytesRef term = next();
- if (term == null) {
- return SeekStatus.END;
- }
- final int cmp = term.compareTo(text);
- if (cmp > 0) {
- return SeekStatus.NOT_FOUND;
- } else if (cmp == 0) {
- return SeekStatus.FOUND;
- }
- }
- }
-
- @Override
- public void seekExact(long ord) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public BytesRef term() throws IOException {
- return term;
- }
-
- @Override
- public long ord() throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public int docFreq() throws IOException {
- return 1;
- }
-
- @Override
- public long totalTermFreq() throws IOException {
- return termFreqs[ord];
- }
-
- @Override
- public final PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException {
- final TVPostingsEnum docsEnum;
- if (reuse != null && reuse instanceof TVPostingsEnum) {
- docsEnum = (TVPostingsEnum) reuse;
- } else {
- docsEnum = new TVPostingsEnum();
- }
-
- docsEnum.reset(
- termFreqs[ord],
- positionIndex[ord],
- positions,
- startOffsets,
- lengths,
- payloads,
- payloadIndex);
- return docsEnum;
- }
-
- @Override
- public ImpactsEnum impacts(int flags) throws IOException {
- final PostingsEnum delegate = postings(null, PostingsEnum.FREQS);
- return new SlowImpactsEnum(delegate);
- }
- }
-
- private static class TVPostingsEnum extends PostingsEnum {
-
- private int doc = -1;
- private int termFreq;
- private int positionIndex;
- private int[] positions;
- private int[] startOffsets;
- private int[] lengths;
- private final BytesRef payload;
- private int[] payloadIndex;
- private int basePayloadOffset;
- private int i;
-
- TVPostingsEnum() {
- payload = new BytesRef();
- }
-
- public void reset(
- int freq,
- int positionIndex,
- int[] positions,
- int[] startOffsets,
- int[] lengths,
- BytesRef payloads,
- int[] payloadIndex) {
- this.termFreq = freq;
- this.positionIndex = positionIndex;
- this.positions = positions;
- this.startOffsets = startOffsets;
- this.lengths = lengths;
- this.basePayloadOffset = payloads.offset;
- this.payload.bytes = payloads.bytes;
- payload.offset = payload.length = 0;
- this.payloadIndex = payloadIndex;
-
- doc = i = -1;
- }
-
- private void checkDoc() {
- if (doc == NO_MORE_DOCS) {
- throw new IllegalStateException("DocsEnum exhausted");
- } else if (doc == -1) {
- throw new IllegalStateException("DocsEnum not started");
- }
- }
-
- private void checkPosition() {
- checkDoc();
- if (i < 0) {
- throw new IllegalStateException("Position enum not started");
- } else if (i >= termFreq) {
- throw new IllegalStateException("Read past last position");
- }
- }
-
- @Override
- public int nextPosition() throws IOException {
- if (doc != 0) {
- throw new IllegalStateException();
- } else if (i >= termFreq - 1) {
- throw new IllegalStateException("Read past last position");
- }
-
- ++i;
-
- if (payloadIndex != null) {
- payload.offset = basePayloadOffset + payloadIndex[positionIndex + i];
- payload.length = payloadIndex[positionIndex + i + 1] - payloadIndex[positionIndex + i];
- }
-
- if (positions == null) {
- return -1;
- } else {
- return positions[positionIndex + i];
- }
- }
-
- @Override
- public int startOffset() throws IOException {
- checkPosition();
- if (startOffsets == null) {
- return -1;
- } else {
- return startOffsets[positionIndex + i];
- }
- }
-
- @Override
- public int endOffset() throws IOException {
- checkPosition();
- if (startOffsets == null) {
- return -1;
- } else {
- return startOffsets[positionIndex + i] + lengths[positionIndex + i];
- }
- }
-
- @Override
- public BytesRef getPayload() throws IOException {
- checkPosition();
- if (payloadIndex == null || payload.length == 0) {
- return null;
- } else {
- return payload;
- }
- }
-
- @Override
- public int freq() throws IOException {
- checkDoc();
- return termFreq;
- }
-
- @Override
- public int docID() {
- return doc;
- }
-
- @Override
- public int nextDoc() throws IOException {
- if (doc == -1) {
- return (doc = 0);
- } else {
- return (doc = NO_MORE_DOCS);
- }
- }
-
- @Override
- public int advance(int target) throws IOException {
- return slowAdvance(target);
- }
-
- @Override
- public long cost() {
- return 1;
- }
- }
-
- private static int sum(int[] arr) {
- int sum = 0;
- for (int el : arr) {
- sum += el;
- }
- return sum;
- }
-
- @Override
- public void checkIntegrity() throws IOException {
- indexReader.checkIntegrity();
- CodecUtil.checksumEntireFile(vectorsStream);
- }
-
- @Override
- public String toString() {
- return getClass().getSimpleName()
- + "(mode="
- + compressionMode
- + ",chunksize="
- + chunkSize
- + ")";
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/package-info.java
deleted file mode 100644
index 53d5f4bd1af..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** Lucene 5.0 compressing format. */
-package org.apache.lucene.backward_codecs.lucene50.compressing;
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/package-info.java
deleted file mode 100644
index 6a01149d255..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** Lucene 5.0 file format. */
-package org.apache.lucene.backward_codecs.lucene50;
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60FieldInfosFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60FieldInfosFormat.java
deleted file mode 100644
index 4e0d7c8e53e..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60FieldInfosFormat.java
+++ /dev/null
@@ -1,372 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene60;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.DocValuesFormat;
-import org.apache.lucene.codecs.FieldInfosFormat;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.DocValuesSkipIndexType;
-import org.apache.lucene.index.DocValuesType;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.VectorEncoding;
-import org.apache.lucene.index.VectorSimilarityFunction;
-import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-
-/**
- * Lucene 6.0 Field Infos format.
- *
- * <p>Field names are stored in the field info file, with suffix <code>.fnm</code>.
- *
- * <p>FieldInfos (.fnm) --&gt; Header,FieldsCount, &lt;FieldName,FieldNumber,
- * FieldBits,DocValuesBits,DocValuesGen,Attributes,DimensionCount,DimensionNumBytes&gt;
- * <sup>FieldsCount</sup>,Footer
- *
- * <p>Data types:
- *
- * <ul>
- * <li>Header --&gt; {@link CodecUtil#checkIndexHeader IndexHeader}
- * <li>FieldsCount --&gt; {@link DataOutput#writeVInt VInt}
- * <li>FieldName --&gt; {@link DataOutput#writeString String}
- * <li>FieldBits, IndexOptions, DocValuesBits --&gt; {@link DataOutput#writeByte Byte}
- * <li>FieldNumber, DimensionCount, DimensionNumBytes --&gt; {@link DataOutput#writeInt VInt}
- * <li>Attributes --&gt; {@link DataOutput#writeMapOfStrings Map&lt;String,String&gt;}
- * <li>DocValuesGen --&gt; {@link DataOutput#writeLong(long) Int64}
- * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
- * </ul>
- *
- * Field Descriptions:
- *
- * <ul>
- * <li>FieldsCount: the number of fields in this file.
- * <li>FieldName: name of the field as a UTF-8 String.
- * <li>FieldNumber: the field's number. Note that unlike previous versions of Lucene, the fields
- * are not numbered implicitly by their order in the file, instead explicitly.
- * <li>FieldBits: a byte containing field options.
- * <ul>
- * <li>The low order bit (0x1) is one for fields that have term vectors stored, and zero for
- * fields without term vectors.
- * <li>If the second lowest order-bit is set (0x2), norms are omitted for the indexed field.
- * <li>If the third lowest-order bit is set (0x4), payloads are stored for the indexed
- * field.
- * </ul>
- * <li>IndexOptions: a byte containing index options.
- * <ul>
- * <li>0: not indexed
- * <li>1: indexed as DOCS_ONLY
- * <li>2: indexed as DOCS_AND_FREQS
- * <li>3: indexed as DOCS_AND_FREQS_AND_POSITIONS
- * <li>4: indexed as DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS
- * </ul>
- * <li>DocValuesBits: a byte containing per-document value types. The type recorded as two
- * four-bit integers, with the high-order bits representing <code>norms</code> options, and
- * the low-order bits representing {@code DocValues} options. Each four-bit integer can be
- * decoded as such:
- * <ul>
- * <li>0: no DocValues for this field.
- * <li>1: NumericDocValues. ({@link DocValuesType#NUMERIC})
- * <li>2: BinaryDocValues. ({@code DocValuesType#BINARY})
- * <li>3: SortedDocValues. ({@code DocValuesType#SORTED})
- * </ul>
- * <li>DocValuesGen is the generation count of the field's DocValues. If this is -1, there are no
- * DocValues updates to that field. Anything above zero means there are updates stored by
- * {@link DocValuesFormat}.
- * <li>Attributes: a key-value map of codec-private attributes.
- * <li>PointDimensionCount, PointNumBytes: these are non-zero only if the field is indexed as
- * points, e.g. using {@link org.apache.lucene.document.LongPoint}
- * </ul>
- *
- * @lucene.experimental
- */
-public final class Lucene60FieldInfosFormat extends FieldInfosFormat {
-
- /** Extension of field infos */
- static final String EXTENSION = "fnm";
-
- // Codec header
- static final String CODEC_NAME = "Lucene60FieldInfos";
- static final int FORMAT_START = 0;
- static final int FORMAT_SOFT_DELETES = 1;
- static final int FORMAT_SELECTIVE_INDEXING = 2;
- static final int FORMAT_CURRENT = FORMAT_SELECTIVE_INDEXING;
-
- // Field flags
- static final byte STORE_TERMVECTOR = 0x1;
- static final byte OMIT_NORMS = 0x2;
- static final byte STORE_PAYLOADS = 0x4;
- static final byte SOFT_DELETES_FIELD = 0x8;
-
- /** Sole constructor. */
- public Lucene60FieldInfosFormat() {}
-
- @Override
- public FieldInfos read(
- Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext context)
- throws IOException {
- final String fileName =
- IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);
- try (ChecksumIndexInput input =
- EndiannessReverserUtil.openChecksumInput(directory, fileName, context)) {
- Throwable priorE = null;
- FieldInfo[] infos = null;
- try {
- int version =
- CodecUtil.checkIndexHeader(
- input,
- Lucene60FieldInfosFormat.CODEC_NAME,
- Lucene60FieldInfosFormat.FORMAT_START,
- Lucene60FieldInfosFormat.FORMAT_CURRENT,
- segmentInfo.getId(),
- segmentSuffix);
-
- infos = readFieldInfos(input, version);
-
- } catch (Throwable exception) {
- priorE = exception;
- } finally {
- CodecUtil.checkFooter(input, priorE);
- }
- return new FieldInfos(infos);
- }
- }
-
- private FieldInfo[] readFieldInfos(IndexInput input, int version) throws IOException {
- final int size = input.readVInt(); // read in the size
- FieldInfo[] infos = new FieldInfo[size];
-
- // previous field's attribute map, we share when possible:
- Map<String, String> lastAttributes = Collections.emptyMap();
-
- for (int i = 0; i < size; i++) {
- String name = input.readString();
- final int fieldNumber = input.readVInt();
- if (fieldNumber < 0) {
- throw new CorruptIndexException(
- "invalid field number for field: " + name + ", fieldNumber=" + fieldNumber, input);
- }
- byte bits = input.readByte();
- boolean storeTermVector = (bits & STORE_TERMVECTOR) != 0;
- boolean omitNorms = (bits & OMIT_NORMS) != 0;
- boolean storePayloads = (bits & STORE_PAYLOADS) != 0;
- boolean isSoftDeletesField = (bits & SOFT_DELETES_FIELD) != 0;
-
- final IndexOptions indexOptions = getIndexOptions(input, input.readByte());
-
- // DV Types are packed in one byte
- final DocValuesType docValuesType = getDocValuesType(input, input.readByte());
- final long dvGen = input.readLong();
- Map<String, String> attributes = input.readMapOfStrings();
- // just use the last field's map if its the same
- if (attributes.equals(lastAttributes)) {
- attributes = lastAttributes;
- }
- lastAttributes = attributes;
- int pointDataDimensionCount = input.readVInt();
- int pointNumBytes;
- int pointIndexDimensionCount = pointDataDimensionCount;
- if (pointDataDimensionCount != 0) {
- if (version >= Lucene60FieldInfosFormat.FORMAT_SELECTIVE_INDEXING) {
- pointIndexDimensionCount = input.readVInt();
- }
- pointNumBytes = input.readVInt();
- } else {
- pointNumBytes = 0;
- }
-
- try {
- infos[i] =
- new FieldInfo(
- name,
- fieldNumber,
- storeTermVector,
- omitNorms,
- storePayloads,
- indexOptions,
- docValuesType,
- DocValuesSkipIndexType.NONE,
- dvGen,
- attributes,
- pointDataDimensionCount,
- pointIndexDimensionCount,
- pointNumBytes,
- 0,
- VectorEncoding.FLOAT32,
- VectorSimilarityFunction.EUCLIDEAN,
- isSoftDeletesField,
- false);
- } catch (IllegalStateException e) {
- throw new CorruptIndexException(
- "invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e);
- }
- }
- return infos;
- }
-
- static {
- // We "mirror" DocValues enum values with the constants below; let's try to ensure if we add a
- // new DocValuesType while this format is
- // still used for writing, we remember to fix this encoding:
- assert DocValuesType.values().length == 6;
- }
-
- private static byte docValuesByte(DocValuesType type) {
- switch (type) {
- case NONE:
- return 0;
- case NUMERIC:
- return 1;
- case BINARY:
- return 2;
- case SORTED:
- return 3;
- case SORTED_SET:
- return 4;
- case SORTED_NUMERIC:
- return 5;
- default:
- // BUG
- throw new AssertionError("unhandled DocValuesType: " + type);
- }
- }
-
- private static DocValuesType getDocValuesType(IndexInput input, byte b) throws IOException {
- switch (b) {
- case 0:
- return DocValuesType.NONE;
- case 1:
- return DocValuesType.NUMERIC;
- case 2:
- return DocValuesType.BINARY;
- case 3:
- return DocValuesType.SORTED;
- case 4:
- return DocValuesType.SORTED_SET;
- case 5:
- return DocValuesType.SORTED_NUMERIC;
- default:
- throw new CorruptIndexException("invalid docvalues byte: " + b, input);
- }
- }
-
- static {
- // We "mirror" IndexOptions enum values with the constants below; let's try to ensure if we add
- // a new IndexOption while this format is
- // still used for writing, we remember to fix this encoding:
- assert IndexOptions.values().length == 5;
- }
-
- private static byte indexOptionsByte(IndexOptions indexOptions) {
- switch (indexOptions) {
- case NONE:
- return 0;
- case DOCS:
- return 1;
- case DOCS_AND_FREQS:
- return 2;
- case DOCS_AND_FREQS_AND_POSITIONS:
- return 3;
- case DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS:
- return 4;
- default:
- // BUG:
- throw new AssertionError("unhandled IndexOptions: " + indexOptions);
- }
- }
-
- private static IndexOptions getIndexOptions(IndexInput input, byte b) throws IOException {
- switch (b) {
- case 0:
- return IndexOptions.NONE;
- case 1:
- return IndexOptions.DOCS;
- case 2:
- return IndexOptions.DOCS_AND_FREQS;
- case 3:
- return IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
- case 4:
- return IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
- default:
- // BUG
- throw new CorruptIndexException("invalid IndexOptions byte: " + b, input);
- }
- }
-
- /**
- * Note: although this format is only used on older versions, we need to keep the write logic in
- * addition to the read logic. It's possible for field infos on older segments to be written to,
- * for example as part of doc values updates.
- */
- @Override
- public void write(
- Directory directory,
- SegmentInfo segmentInfo,
- String segmentSuffix,
- FieldInfos infos,
- IOContext context)
- throws IOException {
- final String fileName =
- IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);
- try (IndexOutput output = EndiannessReverserUtil.createOutput(directory, fileName, context)) {
- CodecUtil.writeIndexHeader(
- output,
- Lucene60FieldInfosFormat.CODEC_NAME,
- Lucene60FieldInfosFormat.FORMAT_CURRENT,
- segmentInfo.getId(),
- segmentSuffix);
- output.writeVInt(infos.size());
- for (FieldInfo fi : infos) {
- fi.checkConsistency();
-
- output.writeString(fi.name);
- output.writeVInt(fi.number);
-
- byte bits = 0x0;
- if (fi.hasTermVectors()) bits |= STORE_TERMVECTOR;
- if (fi.omitsNorms()) bits |= OMIT_NORMS;
- if (fi.hasPayloads()) bits |= STORE_PAYLOADS;
- if (fi.isSoftDeletesField()) bits |= SOFT_DELETES_FIELD;
- output.writeByte(bits);
-
- output.writeByte(indexOptionsByte(fi.getIndexOptions()));
-
- // pack the DV type and hasNorms in one byte
- output.writeByte(docValuesByte(fi.getDocValuesType()));
- output.writeLong(fi.getDocValuesGen());
- output.writeMapOfStrings(fi.attributes());
- output.writeVInt(fi.getPointDimensionCount());
- if (fi.getPointDimensionCount() != 0) {
- output.writeVInt(fi.getPointIndexDimensionCount());
- output.writeVInt(fi.getPointNumBytes());
- }
- }
- CodecUtil.writeFooter(output);
- }
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60PointsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60PointsFormat.java
deleted file mode 100644
index 6d826d26d07..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60PointsFormat.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene60;
-
-import java.io.IOException;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.PointsFormat;
-import org.apache.lucene.codecs.PointsReader;
-import org.apache.lucene.codecs.PointsWriter;
-import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.index.SegmentWriteState;
-
-/**
- * Lucene 6.0 point format, which encodes dimensional values in a block KD-tree structure for fast
- * 1D range and N dimensional shape intersection filtering. See <a
- * href="https://www.cs.duke.edu/~pankaj/publications/papers/bkd-sstd.pdf">this paper</a> for
- * details.
- *
- * <p>This data structure is written as a series of blocks on disk, with an in-memory perfectly
- * balanced binary tree of split values referencing those blocks at the leaves.
- *
- * <p>The <code>.dim</code> file has both blocks and the index split values, for each field. The
- * file starts with {@link CodecUtil#writeIndexHeader}.
- *
- * <p>The blocks are written like this:
- *
- * <ul>
- * <li>count (vInt)
- * <li>delta-docID (vInt) <sup>count</sup> (delta coded docIDs, in sorted order)
- * <li>packedValue<sup>count</sup> (the <code>byte[]</code> value of each dimension packed into a
- * single <code>byte[]</code>)
- * </ul>
- *
- * <p>After all blocks for a field are written, then the index is written:
- *
- * <ul>
- * <li>numDims (vInt)
- * <li>maxPointsInLeafNode (vInt)
- * <li>bytesPerDim (vInt)
- * <li>count (vInt)
- * <li>packed index (byte[])
- * </ul>
- *
- * <p>The packed index uses hierarchical delta and prefix coding to compactly encode the file
- * pointer for all leaf blocks, once the tree is traversed, as well as the split dimension and split
- * value for each inner node of the tree.
- *
- * <p>After all fields blocks + index data are written, {@link CodecUtil#writeFooter} writes the
- * checksum.
- *
- * <p>The <code>.dii</code> file records the file pointer in the <code>.dim</code> file where each
- * field's index data was written. It starts with {@link CodecUtil#writeIndexHeader}, then has:
- *
- * <ul>
- * <li>fieldCount (vInt)
- * <li>(fieldNumber (vInt), fieldFilePointer (vLong))<sup>fieldCount</sup>
- * </ul>
- *
- * <p>After all fields blocks + index data are written, {@link CodecUtil#writeFooter} writes the
- * checksum.
- *
- * @lucene.experimental
- */
-public class Lucene60PointsFormat extends PointsFormat {
-
- static final String DATA_CODEC_NAME = "Lucene60PointsFormatData";
- static final String META_CODEC_NAME = "Lucene60PointsFormatMeta";
-
- /** Filename extension for the leaf blocks */
- public static final String DATA_EXTENSION = "dim";
-
- /** Filename extension for the index per field */
- public static final String INDEX_EXTENSION = "dii";
-
- static final int DATA_VERSION_START = 0;
- static final int DATA_VERSION_CURRENT = DATA_VERSION_START;
-
- static final int INDEX_VERSION_START = 0;
- static final int INDEX_VERSION_CURRENT = INDEX_VERSION_START;
-
- /** Sole constructor */
- public Lucene60PointsFormat() {}
-
- @Override
- public PointsWriter fieldsWriter(SegmentWriteState state) throws IOException {
- throw new UnsupportedOperationException("Old codecs may only be used for reading");
- }
-
- @Override
- public PointsReader fieldsReader(SegmentReadState state) throws IOException {
- return new Lucene60PointsReader(state);
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60PointsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60PointsReader.java
deleted file mode 100644
index 9a7f6bf1547..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60PointsReader.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene60;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.PointsReader;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.PointValues;
-import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.bkd.BKDReader;
-
-/** Reads point values previously written with Lucene60PointsWriter */
-public class Lucene60PointsReader extends PointsReader {
- final IndexInput dataIn;
- final SegmentReadState readState;
- final Map<Integer, PointValues> readers = new HashMap<>();
-
- /** Sole constructor */
- public Lucene60PointsReader(SegmentReadState readState) throws IOException {
- this.readState = readState;
-
- String indexFileName =
- IndexFileNames.segmentFileName(
- readState.segmentInfo.name,
- readState.segmentSuffix,
- Lucene60PointsFormat.INDEX_EXTENSION);
-
- Map<Integer, Long> fieldToFileOffset = new HashMap<>();
-
- // Read index file
- try (ChecksumIndexInput indexIn =
- EndiannessReverserUtil.openChecksumInput(
- readState.directory, indexFileName, readState.context)) {
- Throwable priorE = null;
- try {
- CodecUtil.checkIndexHeader(
- indexIn,
- Lucene60PointsFormat.META_CODEC_NAME,
- Lucene60PointsFormat.INDEX_VERSION_START,
- Lucene60PointsFormat.INDEX_VERSION_CURRENT,
- readState.segmentInfo.getId(),
- readState.segmentSuffix);
- int count = indexIn.readVInt();
- for (int i = 0; i < count; i++) {
- int fieldNumber = indexIn.readVInt();
- long fp = indexIn.readVLong();
- fieldToFileOffset.put(fieldNumber, fp);
- }
- } catch (Throwable t) {
- priorE = t;
- } finally {
- CodecUtil.checkFooter(indexIn, priorE);
- }
- }
-
- String dataFileName =
- IndexFileNames.segmentFileName(
- readState.segmentInfo.name,
- readState.segmentSuffix,
- Lucene60PointsFormat.DATA_EXTENSION);
- boolean success = false;
- dataIn = EndiannessReverserUtil.openInput(readState.directory, dataFileName, readState.context);
- try {
-
- CodecUtil.checkIndexHeader(
- dataIn,
- Lucene60PointsFormat.DATA_CODEC_NAME,
- Lucene60PointsFormat.DATA_VERSION_START,
- Lucene60PointsFormat.DATA_VERSION_START,
- readState.segmentInfo.getId(),
- readState.segmentSuffix);
-
- // NOTE: data file is too costly to verify checksum against all the bytes on open,
- // but for now we at least verify proper structure of the checksum footer: which looks
- // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
- // such as file truncation.
- CodecUtil.retrieveChecksum(dataIn);
-
- for (Map.Entry<Integer, Long> ent : fieldToFileOffset.entrySet()) {
- int fieldNumber = ent.getKey();
- long fp = ent.getValue();
- dataIn.seek(fp);
- PointValues reader = new BKDReader(dataIn, dataIn, dataIn);
- readers.put(fieldNumber, reader);
- }
-
- success = true;
- } finally {
- if (success == false) {
- IOUtils.closeWhileHandlingException(this);
- }
- }
- }
-
- /**
- * Returns the underlying {@link PointValues}.
- *
- * @lucene.internal
- */
- @Override
- public PointValues getValues(String fieldName) {
- FieldInfo fieldInfo = readState.fieldInfos.fieldInfo(fieldName);
- if (fieldInfo == null) {
- throw new IllegalArgumentException("field=\"" + fieldName + "\" is unrecognized");
- }
- if (fieldInfo.getPointDimensionCount() == 0) {
- throw new IllegalArgumentException("field=\"" + fieldName + "\" did not index point values");
- }
-
- return readers.get(fieldInfo.number);
- }
-
- @Override
- public void checkIntegrity() throws IOException {
- CodecUtil.checksumEntireFile(dataIn);
- }
-
- @Override
- public void close() throws IOException {
- dataIn.close();
- // Free up heap:
- readers.clear();
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/package-info.java
deleted file mode 100644
index 9d72a7b984f..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** Lucene 6.0 file format. */
-package org.apache.lucene.backward_codecs.lucene60;
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene70/Lucene70SegmentInfoFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene70/Lucene70SegmentInfoFormat.java
deleted file mode 100644
index 84635f09c4f..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene70/Lucene70SegmentInfoFormat.java
+++ /dev/null
@@ -1,323 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene70;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.Set;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.SegmentInfoFormat;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.SegmentInfos;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.SortedNumericSelector;
-import org.apache.lucene.search.SortedNumericSortField;
-import org.apache.lucene.search.SortedSetSelector;
-import org.apache.lucene.search.SortedSetSortField;
-import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.Version;
-
-/**
- * Lucene 7.0 Segment info format.
- *
- * <p>Files:
- *
- * <ul>
- * <li><code>.si</code>: Header, SegVersion, SegSize, IsCompoundFile, Diagnostics, Files,
- * Attributes, IndexSort, Footer
- * </ul>
- *
- * Data types:
- *
- * <ul>
- * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
- * <li>SegSize --&gt; {@link DataOutput#writeInt Int32}
- * <li>SegVersion --&gt; {@link DataOutput#writeString String}
- * <li>SegMinVersion --&gt; {@link DataOutput#writeString String}
- * <li>Files --&gt; {@link DataOutput#writeSetOfStrings Set&lt;String&gt;}
- * <li>Diagnostics,Attributes --&gt; {@link DataOutput#writeMapOfStrings Map&lt;String,String&gt;}
- * <li>IsCompoundFile --&gt; {@link DataOutput#writeByte Int8}
- * <li>IndexSort --&gt; {@link DataOutput#writeVInt Int32} count, followed by {@code count}
- * SortField
- * <li>SortField --&gt; {@link DataOutput#writeString String} field name, followed by {@link
- * DataOutput#writeVInt Int32} sort type ID, followed by {@link DataOutput#writeByte Int8}
- * indicating reversed sort, followed by a type-specific encoding of the optional missing
- * value
- * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
- * </ul>
- *
- * Field Descriptions:
- *
- * <ul>
- * <li>SegVersion is the code version that created the segment.
- * <li>SegMinVersion is the minimum code version that contributed documents to the segment.
- * <li>SegSize is the number of documents contained in the segment index.
- * <li>IsCompoundFile records whether the segment is written as a compound file or not. If this is
- * -1, the segment is not a compound file. If it is 1, the segment is a compound file.
- * <li>The Diagnostics Map is privately written by {@link IndexWriter}, as a debugging aid, for
- * each segment it creates. It includes metadata like the current Lucene version, OS, Java
- * version, why the segment was created (merge, flush, addIndexes), etc.
- * <li>Files is a list of files referred to by this segment.
- * </ul>
- *
- * @see SegmentInfos
- * @lucene.experimental
- */
-public class Lucene70SegmentInfoFormat extends SegmentInfoFormat {
-
- /** File extension used to store {@link SegmentInfo}. */
- public static final String SI_EXTENSION = "si";
-
- static final String CODEC_NAME = "Lucene70SegmentInfo";
- static final int VERSION_START = 0;
- static final int VERSION_CURRENT = VERSION_START;
-
- /** Sole constructor. */
- public Lucene70SegmentInfoFormat() {}
-
- @Override
- public SegmentInfo read(Directory dir, String segment, byte[] segmentID, IOContext context)
- throws IOException {
- final String fileName =
- IndexFileNames.segmentFileName(segment, "", Lucene70SegmentInfoFormat.SI_EXTENSION);
- try (ChecksumIndexInput input =
- EndiannessReverserUtil.openChecksumInput(dir, fileName, context)) {
- Throwable priorE = null;
- SegmentInfo si = null;
- try {
- CodecUtil.checkIndexHeader(
- input,
- Lucene70SegmentInfoFormat.CODEC_NAME,
- Lucene70SegmentInfoFormat.VERSION_START,
- Lucene70SegmentInfoFormat.VERSION_CURRENT,
- segmentID,
- "");
- si = parseSegmentInfo(dir, input, segment, segmentID);
- } catch (Throwable exception) {
- priorE = exception;
- } finally {
- CodecUtil.checkFooter(input, priorE);
- }
- return si;
- }
- }
-
- private SegmentInfo parseSegmentInfo(
- Directory dir, DataInput input, String segment, byte[] segmentID) throws IOException {
- final Version version = Version.fromBits(input.readInt(), input.readInt(), input.readInt());
- byte hasMinVersion = input.readByte();
- final Version minVersion;
- switch (hasMinVersion) {
- case 0:
- minVersion = null;
- break;
- case 1:
- minVersion = Version.fromBits(input.readInt(), input.readInt(), input.readInt());
- break;
- default:
- throw new CorruptIndexException("Illegal boolean value " + hasMinVersion, input);
- }
-
- final int docCount = input.readInt();
- if (docCount < 0) {
- throw new CorruptIndexException("invalid docCount: " + docCount, input);
- }
- final boolean isCompoundFile = input.readByte() == SegmentInfo.YES;
-
- final Map<String, String> diagnostics = input.readMapOfStrings();
- final Set<String> files = input.readSetOfStrings();
- final Map<String, String> attributes = input.readMapOfStrings();
-
- int numSortFields = input.readVInt();
- Sort indexSort;
- if (numSortFields > 0) {
- SortField[] sortFields = new SortField[numSortFields];
- for (int i = 0; i < numSortFields; i++) {
- String fieldName = input.readString();
- int sortTypeID = input.readVInt();
- SortField.Type sortType;
- SortedSetSelector.Type sortedSetSelector = null;
- SortedNumericSelector.Type sortedNumericSelector = null;
- switch (sortTypeID) {
- case 0:
- sortType = SortField.Type.STRING;
- break;
- case 1:
- sortType = SortField.Type.LONG;
- break;
- case 2:
- sortType = SortField.Type.INT;
- break;
- case 3:
- sortType = SortField.Type.DOUBLE;
- break;
- case 4:
- sortType = SortField.Type.FLOAT;
- break;
- case 5:
- sortType = SortField.Type.STRING;
- byte selector = input.readByte();
- if (selector == 0) {
- sortedSetSelector = SortedSetSelector.Type.MIN;
- } else if (selector == 1) {
- sortedSetSelector = SortedSetSelector.Type.MAX;
- } else if (selector == 2) {
- sortedSetSelector = SortedSetSelector.Type.MIDDLE_MIN;
- } else if (selector == 3) {
- sortedSetSelector = SortedSetSelector.Type.MIDDLE_MAX;
- } else {
- throw new CorruptIndexException(
- "invalid index SortedSetSelector ID: " + selector, input);
- }
- break;
- case 6:
- byte type = input.readByte();
- if (type == 0) {
- sortType = SortField.Type.LONG;
- } else if (type == 1) {
- sortType = SortField.Type.INT;
- } else if (type == 2) {
- sortType = SortField.Type.DOUBLE;
- } else if (type == 3) {
- sortType = SortField.Type.FLOAT;
- } else {
- throw new CorruptIndexException(
- "invalid index SortedNumericSortField type ID: " + type, input);
- }
- byte numericSelector = input.readByte();
- if (numericSelector == 0) {
- sortedNumericSelector = SortedNumericSelector.Type.MIN;
- } else if (numericSelector == 1) {
- sortedNumericSelector = SortedNumericSelector.Type.MAX;
- } else {
- throw new CorruptIndexException(
- "invalid index SortedNumericSelector ID: " + numericSelector, input);
- }
- break;
- default:
- throw new CorruptIndexException(
- "invalid index sort field type ID: " + sortTypeID, input);
- }
- byte b = input.readByte();
- boolean reverse;
- if (b == 0) {
- reverse = true;
- } else if (b == 1) {
- reverse = false;
- } else {
- throw new CorruptIndexException("invalid index sort reverse: " + b, input);
- }
-
- Object missingValue;
- b = input.readByte();
- if (b == 0) {
- missingValue = null;
- } else {
- switch (sortType) {
- case STRING:
- if (b == 1) {
- missingValue = SortField.STRING_LAST;
- } else if (b == 2) {
- missingValue = SortField.STRING_FIRST;
- } else {
- throw new CorruptIndexException("invalid missing value flag: " + b, input);
- }
- break;
- case LONG:
- if (b != 1) {
- throw new CorruptIndexException("invalid missing value flag: " + b, input);
- }
- missingValue = input.readLong();
- break;
- case INT:
- if (b != 1) {
- throw new CorruptIndexException("invalid missing value flag: " + b, input);
- }
- missingValue = input.readInt();
- break;
- case DOUBLE:
- if (b != 1) {
- throw new CorruptIndexException("invalid missing value flag: " + b, input);
- }
- missingValue = Double.longBitsToDouble(input.readLong());
- break;
- case FLOAT:
- if (b != 1) {
- throw new CorruptIndexException("invalid missing value flag: " + b, input);
- }
- missingValue = Float.intBitsToFloat(input.readInt());
- break;
- case CUSTOM:
- case DOC:
- case REWRITEABLE:
- case STRING_VAL:
- case SCORE:
- default:
- throw new AssertionError("unhandled sortType=" + sortType);
- }
- }
-
- if (sortedSetSelector != null) {
- sortFields[i] =
- new SortedSetSortField(fieldName, reverse, sortedSetSelector, missingValue);
- } else if (sortedNumericSelector != null) {
- sortFields[i] =
- new SortedNumericSortField(
- fieldName, sortType, reverse, sortedNumericSelector, missingValue);
- } else {
- sortFields[i] = new SortField(fieldName, sortType, reverse, missingValue);
- }
- }
- indexSort = new Sort(sortFields);
- } else if (numSortFields < 0) {
- throw new CorruptIndexException("invalid index sort field count: " + numSortFields, input);
- } else {
- indexSort = null;
- }
-
- SegmentInfo si =
- new SegmentInfo(
- dir,
- version,
- minVersion,
- segment,
- docCount,
- isCompoundFile,
- false,
- null,
- diagnostics,
- segmentID,
- attributes,
- indexSort);
- si.setFiles(files);
- return si;
- }
-
- @Override
- public void write(Directory dir, SegmentInfo si, IOContext ioContext) throws IOException {
- throw new UnsupportedOperationException("Old formats can't be used for writing");
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene70/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene70/package-info.java
deleted file mode 100644
index 43b9a0554cf..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene70/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Components from the Lucene 7.0 index format. See {@link
- * org.apache.lucene.backward_codecs.lucene86} for an overview of the current index format.
- */
-package org.apache.lucene.backward_codecs.lucene70;
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80Codec.java
deleted file mode 100644
index f4fd646a6ea..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80Codec.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene80;
-
-import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat;
-import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;
-import org.apache.lucene.backward_codecs.lucene60.Lucene60PointsFormat;
-import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.CompoundFormat;
-import org.apache.lucene.codecs.DocValuesFormat;
-import org.apache.lucene.codecs.FieldInfosFormat;
-import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.LiveDocsFormat;
-import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.PointsFormat;
-import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.SegmentInfoFormat;
-import org.apache.lucene.codecs.StoredFieldsFormat;
-import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
-import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
-
-/**
- * Implements the Lucene 8.0 index format.
- *
- * @see org.apache.lucene.backward_codecs.lucene80 package documentation for file format details.
- * @lucene.experimental
- */
-public class Lucene80Codec extends Codec {
- private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat();
- private final FieldInfosFormat fieldInfosFormat = new Lucene60FieldInfosFormat();
- private final SegmentInfoFormat segmentInfosFormat = new Lucene70SegmentInfoFormat();
- private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
- private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
-
- private final PostingsFormat postingsFormat =
- new PerFieldPostingsFormat() {
- @Override
- public PostingsFormat getPostingsFormatForField(String field) {
- throw new UnsupportedOperationException("Old codecs can't be used for writing");
- }
- };
-
- private final DocValuesFormat docValuesFormat =
- new PerFieldDocValuesFormat() {
- @Override
- public DocValuesFormat getDocValuesFormatForField(String field) {
- return defaultDVFormat;
- }
- };
- private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene80");
-
- private final StoredFieldsFormat storedFieldsFormat;
-
- /** Instantiates a new codec. */
- public Lucene80Codec() {
- super("Lucene80");
- this.storedFieldsFormat = new Lucene50StoredFieldsFormat();
- }
-
- @Override
- public final StoredFieldsFormat storedFieldsFormat() {
- return storedFieldsFormat;
- }
-
- @Override
- public TermVectorsFormat termVectorsFormat() {
- return vectorsFormat;
- }
-
- @Override
- public final PostingsFormat postingsFormat() {
- return postingsFormat;
- }
-
- @Override
- public final FieldInfosFormat fieldInfosFormat() {
- return fieldInfosFormat;
- }
-
- @Override
- public final SegmentInfoFormat segmentInfoFormat() {
- return segmentInfosFormat;
- }
-
- @Override
- public final LiveDocsFormat liveDocsFormat() {
- return liveDocsFormat;
- }
-
- @Override
- public final CompoundFormat compoundFormat() {
- return compoundFormat;
- }
-
- @Override
- public final PointsFormat pointsFormat() {
- return new Lucene60PointsFormat();
- }
-
- @Override
- public final DocValuesFormat docValuesFormat() {
- return docValuesFormat;
- }
-
- private final NormsFormat normsFormat = new Lucene80NormsFormat();
-
- @Override
- public final NormsFormat normsFormat() {
- return normsFormat;
- }
-
- @Override
- public final KnnVectorsFormat knnVectorsFormat() {
- return KnnVectorsFormat.EMPTY;
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80NormsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80NormsFormat.java
deleted file mode 100644
index 28b191c2d6b..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80NormsFormat.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene80;
-
-import java.io.IOException;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.NormsConsumer;
-import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.NormsProducer;
-import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.store.DataOutput;
-
-/**
- * Lucene 8.0 Score normalization format.
- *
- * <p>Encodes normalization values by encoding each value with the minimum number of bytes needed to
- * represent the range (which can be zero).
- *
- * <p>Files:
- *
- * <ol>
- * <li><code>.nvd</code>: Norms data
- * <li><code>.nvm</code>: Norms metadata
- * </ol>
- *
- * <ol>
- * <li><a id="nvm"></a>
- * <p>The Norms metadata or .nvm file.
- * <p>For each norms field, this stores metadata, such as the offset into the Norms data
- * (.nvd)
- * <p>Norms metadata (.dvm) --&gt; Header,&lt;Entry&gt;<sup>NumFields</sup>,Footer
- * <ul>
- * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
- * <li>Entry --&gt; FieldNumber, DocsWithFieldAddress, DocsWithFieldLength,
- * NumDocsWithField, BytesPerNorm, NormsAddress
- * <li>FieldNumber --&gt; {@link DataOutput#writeInt Int32}
- * <li>DocsWithFieldAddress --&gt; {@link DataOutput#writeLong Int64}
- * <li>DocsWithFieldLength --&gt; {@link DataOutput#writeLong Int64}
- * <li>NumDocsWithField --&gt; {@link DataOutput#writeInt Int32}
- * <li>BytesPerNorm --&gt; {@link DataOutput#writeByte byte}
- * <li>NormsAddress --&gt; {@link DataOutput#writeLong Int64}
- * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
- * </ul>
- * <p>FieldNumber of -1 indicates the end of metadata.
- * <p>NormsAddress is the pointer to the start of the data in the norms data (.nvd), or the
- * singleton value when BytesPerValue = 0. If BytesPerValue is different from 0 then there are
- * NumDocsWithField values to read at that offset.
- * <p>DocsWithFieldAddress is the pointer to the start of the bit set containing documents
- * that have a norm in the norms data (.nvd), or -2 if no documents have a norm value, or -1
- * if all documents have a norm value.
- * <p>DocsWithFieldLength is the number of bytes used to encode the set of documents that have
- * a norm.
- * <li><a id="nvd"></a>
- * <p>The Norms data or .nvd file.
- * <p>For each Norms field, this stores the actual per-document data (the heavy-lifting)
- * <p>Norms data (.nvd) --&gt; Header,&lt; Data &gt;<sup>NumFields</sup>,Footer
- * <ul>
- * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
- * <li>DocsWithFieldData --&gt; {@link IndexedDISI#writeBitSet Bit set of MaxDoc bits}
- * <li>NormsData --&gt; {@link DataOutput#writeByte(byte) byte}<sup>NumDocsWithField *
- * BytesPerValue</sup>
- * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
- * </ul>
- * </ol>
- *
- * @lucene.experimental
- */
-public class Lucene80NormsFormat extends NormsFormat {
-
- /** Sole Constructor */
- public Lucene80NormsFormat() {}
-
- @Override
- public NormsConsumer normsConsumer(SegmentWriteState state) throws IOException {
- throw new UnsupportedOperationException("Old codecs may only be used for reading");
- }
-
- @Override
- public NormsProducer normsProducer(SegmentReadState state) throws IOException {
- return new Lucene80NormsProducer(
- state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
- }
-
- static final String DATA_CODEC = "Lucene80NormsData";
- static final String DATA_EXTENSION = "nvd";
- static final String METADATA_CODEC = "Lucene80NormsMetadata";
- static final String METADATA_EXTENSION = "nvm";
- static final int VERSION_START = 0;
- static final int VERSION_CURRENT = VERSION_START;
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80NormsProducer.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80NormsProducer.java
deleted file mode 100644
index cf4d8df66ee..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80NormsProducer.java
+++ /dev/null
@@ -1,489 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene80;
-
-import static org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat.VERSION_CURRENT;
-import static org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat.VERSION_START;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.NormsProducer;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.RandomAccessInput;
-import org.apache.lucene.util.IOUtils;
-
-/** Reader for {@link Lucene80NormsFormat} */
-final class Lucene80NormsProducer extends NormsProducer implements Cloneable {
- // metadata maps (just file pointers and minimal stuff)
- private final Map<Integer, NormsEntry> norms = new HashMap<>();
- private final int maxDoc;
- private IndexInput data;
- private boolean merging;
- private Map<Integer, IndexInput> disiInputs;
- private Map<Integer, RandomAccessInput> disiJumpTables;
- private Map<Integer, RandomAccessInput> dataInputs;
-
- Lucene80NormsProducer(
- SegmentReadState state,
- String dataCodec,
- String dataExtension,
- String metaCodec,
- String metaExtension)
- throws IOException {
- maxDoc = state.segmentInfo.maxDoc();
- String metaName =
- IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
- int version = -1;
-
- // read in the entries from the metadata file.
- try (ChecksumIndexInput in =
- EndiannessReverserUtil.openChecksumInput(state.directory, metaName, state.context)) {
- Throwable priorE = null;
- try {
- version =
- CodecUtil.checkIndexHeader(
- in,
- metaCodec,
- VERSION_START,
- VERSION_CURRENT,
- state.segmentInfo.getId(),
- state.segmentSuffix);
- readFields(in, state.fieldInfos);
- } catch (Throwable exception) {
- priorE = exception;
- } finally {
- CodecUtil.checkFooter(in, priorE);
- }
- }
-
- String dataName =
- IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
- data = EndiannessReverserUtil.openInput(state.directory, dataName, state.context);
- boolean success = false;
- try {
- final int version2 =
- CodecUtil.checkIndexHeader(
- data,
- dataCodec,
- VERSION_START,
- VERSION_CURRENT,
- state.segmentInfo.getId(),
- state.segmentSuffix);
- if (version != version2) {
- throw new CorruptIndexException(
- "Format versions mismatch: meta=" + version + ",data=" + version2, data);
- }
-
- // NOTE: data file is too costly to verify checksum against all the bytes on open,
- // but for now we at least verify proper structure of the checksum footer: which looks
- // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
- // such as file truncation.
- CodecUtil.retrieveChecksum(data);
-
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(this.data);
- }
- }
- }
-
- @Override
- public NormsProducer getMergeInstance() {
- Lucene80NormsProducer clone;
- try {
- clone = (Lucene80NormsProducer) super.clone();
- } catch (CloneNotSupportedException e) {
- // cannot happen
- throw new RuntimeException(e);
- }
- clone.data = data.clone();
- clone.disiInputs = new HashMap<>();
- clone.disiJumpTables = new HashMap<>();
- clone.dataInputs = new HashMap<>();
- clone.merging = true;
- return clone;
- }
-
- static class NormsEntry {
- byte denseRankPower;
- byte bytesPerNorm;
- long docsWithFieldOffset;
- long docsWithFieldLength;
- short jumpTableEntryCount;
- int numDocsWithField;
- long normsOffset;
- }
-
- abstract static class DenseNormsIterator extends NumericDocValues {
-
- final int maxDoc;
- int doc = -1;
-
- DenseNormsIterator(int maxDoc) {
- this.maxDoc = maxDoc;
- }
-
- @Override
- public int docID() {
- return doc;
- }
-
- @Override
- public int nextDoc() throws IOException {
- return advance(doc + 1);
- }
-
- @Override
- public int advance(int target) throws IOException {
- if (target >= maxDoc) {
- return doc = NO_MORE_DOCS;
- }
- return doc = target;
- }
-
- @Override
- public boolean advanceExact(int target) throws IOException {
- this.doc = target;
- return true;
- }
-
- @Override
- public long cost() {
- return maxDoc;
- }
- }
-
- abstract static class SparseNormsIterator extends NumericDocValues {
-
- final IndexedDISI disi;
-
- SparseNormsIterator(IndexedDISI disi) {
- this.disi = disi;
- }
-
- @Override
- public int docID() {
- return disi.docID();
- }
-
- @Override
- public int nextDoc() throws IOException {
- return disi.nextDoc();
- }
-
- @Override
- public int advance(int target) throws IOException {
- return disi.advance(target);
- }
-
- @Override
- public boolean advanceExact(int target) throws IOException {
- return disi.advanceExact(target);
- }
-
- @Override
- public long cost() {
- return disi.cost();
- }
- }
-
- private void readFields(IndexInput meta, FieldInfos infos) throws IOException {
- for (int fieldNumber = meta.readInt(); fieldNumber != -1; fieldNumber = meta.readInt()) {
- FieldInfo info = infos.fieldInfo(fieldNumber);
- if (info == null) {
- throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);
- } else if (!info.hasNorms()) {
- throw new CorruptIndexException("Invalid field: " + info.name, meta);
- }
- NormsEntry entry = new NormsEntry();
- entry.docsWithFieldOffset = meta.readLong();
- entry.docsWithFieldLength = meta.readLong();
- entry.jumpTableEntryCount = meta.readShort();
- entry.denseRankPower = meta.readByte();
- entry.numDocsWithField = meta.readInt();
- entry.bytesPerNorm = meta.readByte();
- switch (entry.bytesPerNorm) {
- case 0:
- case 1:
- case 2:
- case 4:
- case 8:
- break;
- default:
- throw new CorruptIndexException(
- "Invalid bytesPerValue: " + entry.bytesPerNorm + ", field: " + info.name, meta);
- }
- entry.normsOffset = meta.readLong();
- norms.put(info.number, entry);
- }
- }
-
- private RandomAccessInput getDataInput(FieldInfo field, NormsEntry entry) throws IOException {
- RandomAccessInput slice = null;
- if (merging) {
- slice = dataInputs.get(field.number);
- }
- if (slice == null) {
- slice =
- data.randomAccessSlice(
- entry.normsOffset, entry.numDocsWithField * (long) entry.bytesPerNorm);
- if (merging) {
- dataInputs.put(field.number, slice);
- }
- }
- return slice;
- }
-
- private IndexInput getDisiInput(FieldInfo field, NormsEntry entry) throws IOException {
- if (merging == false) {
- return IndexedDISI.createBlockSlice(
- data,
- "docs",
- entry.docsWithFieldOffset,
- entry.docsWithFieldLength,
- entry.jumpTableEntryCount);
- }
-
- IndexInput in = disiInputs.get(field.number);
- if (in == null) {
- in =
- IndexedDISI.createBlockSlice(
- data,
- "docs",
- entry.docsWithFieldOffset,
- entry.docsWithFieldLength,
- entry.jumpTableEntryCount);
- disiInputs.put(field.number, in);
- }
-
- final IndexInput inF = in; // same as in but final
-
- // Wrap so that reads can be interleaved from the same thread if two
- // norms instances are pulled and consumed in parallel. Merging usually
- // doesn't need this feature but CheckIndex might, plus we need merge
- // instances to behave well and not be trappy.
- return new IndexInput("docs") {
-
- long offset = 0;
-
- @Override
- public void readBytes(byte[] b, int off, int len) throws IOException {
- inF.seek(offset);
- offset += len;
- inF.readBytes(b, off, len);
- }
-
- @Override
- public byte readByte() throws IOException {
- throw new UnsupportedOperationException("Unused by IndexedDISI");
- }
-
- @Override
- public IndexInput slice(String sliceDescription, long offset, long length)
- throws IOException {
- throw new UnsupportedOperationException("Unused by IndexedDISI");
- }
-
- @Override
- public short readShort() throws IOException {
- inF.seek(offset);
- offset += Short.BYTES;
- return inF.readShort();
- }
-
- @Override
- public long readLong() throws IOException {
- inF.seek(offset);
- offset += Long.BYTES;
- return inF.readLong();
- }
-
- @Override
- public void seek(long pos) throws IOException {
- offset = pos;
- }
-
- @Override
- public long length() {
- throw new UnsupportedOperationException("Unused by IndexedDISI");
- }
-
- @Override
- public long getFilePointer() {
- return offset;
- }
-
- @Override
- public void close() throws IOException {
- throw new UnsupportedOperationException("Unused by IndexedDISI");
- }
- };
- }
-
- private RandomAccessInput getDisiJumpTable(FieldInfo field, NormsEntry entry) throws IOException {
- RandomAccessInput jumpTable = null;
- if (merging) {
- jumpTable = disiJumpTables.get(field.number);
- }
- if (jumpTable == null) {
- jumpTable =
- IndexedDISI.createJumpTable(
- data,
- entry.docsWithFieldOffset,
- entry.docsWithFieldLength,
- entry.jumpTableEntryCount);
- if (merging) {
- disiJumpTables.put(field.number, jumpTable);
- }
- }
- return jumpTable;
- }
-
- @Override
- public NumericDocValues getNorms(FieldInfo field) throws IOException {
- final NormsEntry entry = norms.get(field.number);
- if (entry.docsWithFieldOffset == -2) {
- // empty
- return DocValues.emptyNumeric();
- } else if (entry.docsWithFieldOffset == -1) {
- // dense
- if (entry.bytesPerNorm == 0) {
- return new DenseNormsIterator(maxDoc) {
- @Override
- public long longValue() throws IOException {
- return entry.normsOffset;
- }
- };
- }
- final RandomAccessInput slice = getDataInput(field, entry);
- switch (entry.bytesPerNorm) {
- case 1:
- return new DenseNormsIterator(maxDoc) {
- @Override
- public long longValue() throws IOException {
- return slice.readByte(doc);
- }
- };
- case 2:
- return new DenseNormsIterator(maxDoc) {
- @Override
- public long longValue() throws IOException {
- return slice.readShort(((long) doc) << 1);
- }
- };
- case 4:
- return new DenseNormsIterator(maxDoc) {
- @Override
- public long longValue() throws IOException {
- return slice.readInt(((long) doc) << 2);
- }
- };
- case 8:
- return new DenseNormsIterator(maxDoc) {
- @Override
- public long longValue() throws IOException {
- return slice.readLong(((long) doc) << 3);
- }
- };
- default:
- // should not happen, we already validate bytesPerNorm in readFields
- throw new AssertionError();
- }
- } else {
- // sparse
- final IndexInput disiInput = getDisiInput(field, entry);
- final RandomAccessInput disiJumpTable = getDisiJumpTable(field, entry);
- final IndexedDISI disi =
- new IndexedDISI(
- disiInput,
- disiJumpTable,
- entry.jumpTableEntryCount,
- entry.denseRankPower,
- entry.numDocsWithField);
-
- if (entry.bytesPerNorm == 0) {
- return new SparseNormsIterator(disi) {
- @Override
- public long longValue() throws IOException {
- return entry.normsOffset;
- }
- };
- }
- final RandomAccessInput slice = getDataInput(field, entry);
- switch (entry.bytesPerNorm) {
- case 1:
- return new SparseNormsIterator(disi) {
- @Override
- public long longValue() throws IOException {
- return slice.readByte(disi.index());
- }
- };
- case 2:
- return new SparseNormsIterator(disi) {
- @Override
- public long longValue() throws IOException {
- return slice.readShort(((long) disi.index()) << 1);
- }
- };
- case 4:
- return new SparseNormsIterator(disi) {
- @Override
- public long longValue() throws IOException {
- return slice.readInt(((long) disi.index()) << 2);
- }
- };
- case 8:
- return new SparseNormsIterator(disi) {
- @Override
- public long longValue() throws IOException {
- return slice.readLong(((long) disi.index()) << 3);
- }
- };
- default:
- // should not happen, we already validate bytesPerNorm in readFields
- throw new AssertionError();
- }
- }
- }
-
- @Override
- public void close() throws IOException {
- data.close();
- }
-
- @Override
- public void checkIntegrity() throws IOException {
- CodecUtil.checksumEntireFile(data);
- }
-
- @Override
- public String toString() {
- return getClass().getSimpleName() + "(fields=" + norms.size() + ")";
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/ForDeltaUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/ForDeltaUtil.java
deleted file mode 100644
index dd5f1f7a6a3..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/ForDeltaUtil.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene84;
-
-import java.io.IOException;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.util.packed.PackedInts;
-
-/** Utility class to encode/decode increasing sequences of 128 integers. */
-public class ForDeltaUtil {
-
- // IDENTITY_PLUS_ONE[i] == i+1
- private static final long[] IDENTITY_PLUS_ONE = new long[ForUtil.BLOCK_SIZE];
-
- static {
- for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {
- IDENTITY_PLUS_ONE[i] = i + 1;
- }
- }
-
- private static void prefixSumOfOnes(long[] arr, long base) {
- System.arraycopy(IDENTITY_PLUS_ONE, 0, arr, 0, ForUtil.BLOCK_SIZE);
- // This loop gets auto-vectorized
- for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {
- arr[i] += base;
- }
- }
-
- private final ForUtil forUtil;
-
- ForDeltaUtil(ForUtil forUtil) {
- this.forUtil = forUtil;
- }
-
- /**
- * Encode deltas of a strictly monotonically increasing sequence of integers. The provided {@code
- * longs} are expected to be deltas between consecutive values.
- */
- void encodeDeltas(long[] longs, DataOutput out) throws IOException {
- if (longs[0] == 1 && PForUtil.allEqual(longs)) { // happens with very dense postings
- out.writeByte((byte) 0);
- } else {
- long or = 0;
- for (long l : longs) {
- or |= l;
- }
- assert or != 0;
- final int bitsPerValue = PackedInts.bitsRequired(or);
- out.writeByte((byte) bitsPerValue);
- forUtil.encode(longs, bitsPerValue, out);
- }
- }
-
- /** Decode deltas, compute the prefix sum and add {@code base} to all decoded longs. */
- void decodeAndPrefixSum(DataInput in, long base, long[] longs) throws IOException {
- final int bitsPerValue = Byte.toUnsignedInt(in.readByte());
- if (bitsPerValue == 0) {
- prefixSumOfOnes(longs, base);
- } else {
- forUtil.decodeAndPrefixSum(bitsPerValue, in, base, longs);
- }
- }
-
- /** Skip a sequence of 128 longs. */
- void skip(DataInput in) throws IOException {
- final int bitsPerValue = Byte.toUnsignedInt(in.readByte());
- if (bitsPerValue != 0) {
- in.skipBytes(forUtil.numBytes(bitsPerValue));
- }
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/ForUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/ForUtil.java
deleted file mode 100644
index f7d1f6c501b..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/ForUtil.java
+++ /dev/null
@@ -1,1159 +0,0 @@
-// This file has been automatically generated, DO NOT EDIT
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene84;
-
-import java.io.IOException;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.DataOutput;
-
-// Inspired from https://fulmicoton.com/posts/bitpacking/
-// Encodes multiple integers in a long to get SIMD-like speedups.
-// If bitsPerValue <= 8 then we pack 8 ints per long
-// else if bitsPerValue <= 16 we pack 4 ints per long
-// else we pack 2 ints per long
-final class ForUtil {
-
- static final int BLOCK_SIZE = 128;
- private static final int BLOCK_SIZE_LOG2 = 7;
-
- private static long expandMask32(long mask32) {
- return mask32 | (mask32 << 32);
- }
-
- private static long expandMask16(long mask16) {
- return expandMask32(mask16 | (mask16 << 16));
- }
-
- private static long expandMask8(long mask8) {
- return expandMask16(mask8 | (mask8 << 8));
- }
-
- private static long mask32(int bitsPerValue) {
- return expandMask32((1L << bitsPerValue) - 1);
- }
-
- private static long mask16(int bitsPerValue) {
- return expandMask16((1L << bitsPerValue) - 1);
- }
-
- private static long mask8(int bitsPerValue) {
- return expandMask8((1L << bitsPerValue) - 1);
- }
-
- private static void expand8(long[] arr) {
- for (int i = 0; i < 16; ++i) {
- long l = arr[i];
- arr[i] = (l >>> 56) & 0xFFL;
- arr[16 + i] = (l >>> 48) & 0xFFL;
- arr[32 + i] = (l >>> 40) & 0xFFL;
- arr[48 + i] = (l >>> 32) & 0xFFL;
- arr[64 + i] = (l >>> 24) & 0xFFL;
- arr[80 + i] = (l >>> 16) & 0xFFL;
- arr[96 + i] = (l >>> 8) & 0xFFL;
- arr[112 + i] = l & 0xFFL;
- }
- }
-
- private static void expand8To32(long[] arr) {
- for (int i = 0; i < 16; ++i) {
- long l = arr[i];
- arr[i] = (l >>> 24) & 0x000000FF000000FFL;
- arr[16 + i] = (l >>> 16) & 0x000000FF000000FFL;
- arr[32 + i] = (l >>> 8) & 0x000000FF000000FFL;
- arr[48 + i] = l & 0x000000FF000000FFL;
- }
- }
-
- private static void collapse8(long[] arr) {
- for (int i = 0; i < 16; ++i) {
- arr[i] =
- (arr[i] << 56)
- | (arr[16 + i] << 48)
- | (arr[32 + i] << 40)
- | (arr[48 + i] << 32)
- | (arr[64 + i] << 24)
- | (arr[80 + i] << 16)
- | (arr[96 + i] << 8)
- | arr[112 + i];
- }
- }
-
- private static void expand16(long[] arr) {
- for (int i = 0; i < 32; ++i) {
- long l = arr[i];
- arr[i] = (l >>> 48) & 0xFFFFL;
- arr[32 + i] = (l >>> 32) & 0xFFFFL;
- arr[64 + i] = (l >>> 16) & 0xFFFFL;
- arr[96 + i] = l & 0xFFFFL;
- }
- }
-
- private static void expand16To32(long[] arr) {
- for (int i = 0; i < 32; ++i) {
- long l = arr[i];
- arr[i] = (l >>> 16) & 0x0000FFFF0000FFFFL;
- arr[32 + i] = l & 0x0000FFFF0000FFFFL;
- }
- }
-
- private static void collapse16(long[] arr) {
- for (int i = 0; i < 32; ++i) {
- arr[i] = (arr[i] << 48) | (arr[32 + i] << 32) | (arr[64 + i] << 16) | arr[96 + i];
- }
- }
-
- private static void expand32(long[] arr) {
- for (int i = 0; i < 64; ++i) {
- long l = arr[i];
- arr[i] = l >>> 32;
- arr[64 + i] = l & 0xFFFFFFFFL;
- }
- }
-
- private static void collapse32(long[] arr) {
- for (int i = 0; i < 64; ++i) {
- arr[i] = (arr[i] << 32) | arr[64 + i];
- }
- }
-
- private static void prefixSum8(long[] arr, long base) {
- expand8To32(arr);
- prefixSum32(arr, base);
- }
-
- private static void prefixSum16(long[] arr, long base) {
- // We need to move to the next primitive size to avoid overflows
- expand16To32(arr);
- prefixSum32(arr, base);
- }
-
- private static void prefixSum32(long[] arr, long base) {
- arr[0] += base << 32;
- innerPrefixSum32(arr);
- expand32(arr);
- final long l = arr[BLOCK_SIZE / 2 - 1];
- for (int i = BLOCK_SIZE / 2; i < BLOCK_SIZE; ++i) {
- arr[i] += l;
- }
- }
-
- // For some reason unrolling seems to help
- private static void innerPrefixSum32(long[] arr) {
- arr[1] += arr[0];
- arr[2] += arr[1];
- arr[3] += arr[2];
- arr[4] += arr[3];
- arr[5] += arr[4];
- arr[6] += arr[5];
- arr[7] += arr[6];
- arr[8] += arr[7];
- arr[9] += arr[8];
- arr[10] += arr[9];
- arr[11] += arr[10];
- arr[12] += arr[11];
- arr[13] += arr[12];
- arr[14] += arr[13];
- arr[15] += arr[14];
- arr[16] += arr[15];
- arr[17] += arr[16];
- arr[18] += arr[17];
- arr[19] += arr[18];
- arr[20] += arr[19];
- arr[21] += arr[20];
- arr[22] += arr[21];
- arr[23] += arr[22];
- arr[24] += arr[23];
- arr[25] += arr[24];
- arr[26] += arr[25];
- arr[27] += arr[26];
- arr[28] += arr[27];
- arr[29] += arr[28];
- arr[30] += arr[29];
- arr[31] += arr[30];
- arr[32] += arr[31];
- arr[33] += arr[32];
- arr[34] += arr[33];
- arr[35] += arr[34];
- arr[36] += arr[35];
- arr[37] += arr[36];
- arr[38] += arr[37];
- arr[39] += arr[38];
- arr[40] += arr[39];
- arr[41] += arr[40];
- arr[42] += arr[41];
- arr[43] += arr[42];
- arr[44] += arr[43];
- arr[45] += arr[44];
- arr[46] += arr[45];
- arr[47] += arr[46];
- arr[48] += arr[47];
- arr[49] += arr[48];
- arr[50] += arr[49];
- arr[51] += arr[50];
- arr[52] += arr[51];
- arr[53] += arr[52];
- arr[54] += arr[53];
- arr[55] += arr[54];
- arr[56] += arr[55];
- arr[57] += arr[56];
- arr[58] += arr[57];
- arr[59] += arr[58];
- arr[60] += arr[59];
- arr[61] += arr[60];
- arr[62] += arr[61];
- arr[63] += arr[62];
- }
-
- private static void readLELongs(DataInput in, long[] dst, int offset, int length)
- throws IOException {
- in.readLongs(dst, offset, length);
- for (int i = 0; i < length; ++i) {
- dst[offset + i] = Long.reverseBytes(dst[offset + i]);
- }
- }
-
- private final long[] tmp = new long[BLOCK_SIZE / 2];
-
- /** Encode 128 integers from {@code longs} into {@code out}. */
- void encode(long[] longs, int bitsPerValue, DataOutput out) throws IOException {
- final int nextPrimitive;
- final int numLongs;
- if (bitsPerValue <= 8) {
- nextPrimitive = 8;
- numLongs = BLOCK_SIZE / 8;
- collapse8(longs);
- } else if (bitsPerValue <= 16) {
- nextPrimitive = 16;
- numLongs = BLOCK_SIZE / 4;
- collapse16(longs);
- } else {
- nextPrimitive = 32;
- numLongs = BLOCK_SIZE / 2;
- collapse32(longs);
- }
-
- final int numLongsPerShift = bitsPerValue * 2;
- int idx = 0;
- int shift = nextPrimitive - bitsPerValue;
- for (int i = 0; i < numLongsPerShift; ++i) {
- tmp[i] = longs[idx++] << shift;
- }
- for (shift = shift - bitsPerValue; shift >= 0; shift -= bitsPerValue) {
- for (int i = 0; i < numLongsPerShift; ++i) {
- tmp[i] |= longs[idx++] << shift;
- }
- }
-
- final int remainingBitsPerLong = shift + bitsPerValue;
- final long maskRemainingBitsPerLong;
- if (nextPrimitive == 8) {
- maskRemainingBitsPerLong = MASKS8[remainingBitsPerLong];
- } else if (nextPrimitive == 16) {
- maskRemainingBitsPerLong = MASKS16[remainingBitsPerLong];
- } else {
- maskRemainingBitsPerLong = MASKS32[remainingBitsPerLong];
- }
-
- int tmpIdx = 0;
- int remainingBitsPerValue = bitsPerValue;
- while (idx < numLongs) {
- if (remainingBitsPerValue >= remainingBitsPerLong) {
- remainingBitsPerValue -= remainingBitsPerLong;
- tmp[tmpIdx++] |= (longs[idx] >>> remainingBitsPerValue) & maskRemainingBitsPerLong;
- if (remainingBitsPerValue == 0) {
- idx++;
- remainingBitsPerValue = bitsPerValue;
- }
- } else {
- final long mask1, mask2;
- if (nextPrimitive == 8) {
- mask1 = MASKS8[remainingBitsPerValue];
- mask2 = MASKS8[remainingBitsPerLong - remainingBitsPerValue];
- } else if (nextPrimitive == 16) {
- mask1 = MASKS16[remainingBitsPerValue];
- mask2 = MASKS16[remainingBitsPerLong - remainingBitsPerValue];
- } else {
- mask1 = MASKS32[remainingBitsPerValue];
- mask2 = MASKS32[remainingBitsPerLong - remainingBitsPerValue];
- }
- tmp[tmpIdx] |= (longs[idx++] & mask1) << (remainingBitsPerLong - remainingBitsPerValue);
- remainingBitsPerValue = bitsPerValue - remainingBitsPerLong + remainingBitsPerValue;
- tmp[tmpIdx++] |= (longs[idx] >>> remainingBitsPerValue) & mask2;
- }
- }
-
- for (int i = 0; i < numLongsPerShift; ++i) {
- // Java longs are big endian and we want to read little endian longs, so we need to reverse
- // bytes
- long l = Long.reverseBytes(tmp[i]);
- out.writeLong(l);
- }
- }
-
- /** Number of bytes required to encode 128 integers of {@code bitsPerValue} bits per value. */
- int numBytes(int bitsPerValue) throws IOException {
- return bitsPerValue << (BLOCK_SIZE_LOG2 - 3);
- }
-
- private static void decodeSlow(int bitsPerValue, DataInput in, long[] tmp, long[] longs)
- throws IOException {
- final int numLongs = bitsPerValue << 1;
- readLELongs(in, tmp, 0, numLongs);
- final long mask = MASKS32[bitsPerValue];
- int longsIdx = 0;
- int shift = 32 - bitsPerValue;
- for (; shift >= 0; shift -= bitsPerValue) {
- shiftLongs(tmp, numLongs, longs, longsIdx, shift, mask);
- longsIdx += numLongs;
- }
- final int remainingBitsPerLong = shift + bitsPerValue;
- final long mask32RemainingBitsPerLong = MASKS32[remainingBitsPerLong];
- int tmpIdx = 0;
- int remainingBits = remainingBitsPerLong;
- for (; longsIdx < BLOCK_SIZE / 2; ++longsIdx) {
- int b = bitsPerValue - remainingBits;
- long l = (tmp[tmpIdx++] & MASKS32[remainingBits]) << b;
- while (b >= remainingBitsPerLong) {
- b -= remainingBitsPerLong;
- l |= (tmp[tmpIdx++] & mask32RemainingBitsPerLong) << b;
- }
- if (b > 0) {
- l |= (tmp[tmpIdx] >>> (remainingBitsPerLong - b)) & MASKS32[b];
- remainingBits = remainingBitsPerLong - b;
- } else {
- remainingBits = remainingBitsPerLong;
- }
- longs[longsIdx] = l;
- }
- }
-
- /**
- * The pattern that this shiftLongs method applies is recognized by the C2 compiler, which
- * generates SIMD instructions for it in order to shift multiple longs at once.
- */
- private static void shiftLongs(long[] a, int count, long[] b, int bi, int shift, long mask) {
- for (int i = 0; i < count; ++i) {
- b[bi + i] = (a[i] >>> shift) & mask;
- }
- }
-
- private static final long[] MASKS8 = new long[8];
- private static final long[] MASKS16 = new long[16];
- private static final long[] MASKS32 = new long[32];
-
- static {
- for (int i = 0; i < 8; ++i) {
- MASKS8[i] = mask8(i);
- }
- for (int i = 0; i < 16; ++i) {
- MASKS16[i] = mask16(i);
- }
- for (int i = 0; i < 32; ++i) {
- MASKS32[i] = mask32(i);
- }
- }
-
- // mark values in array as final longs to avoid the cost of reading array, arrays should only be
- // used when the idx is a variable
- private static final long MASK8_1 = MASKS8[1];
- private static final long MASK8_2 = MASKS8[2];
- private static final long MASK8_3 = MASKS8[3];
- private static final long MASK8_4 = MASKS8[4];
- private static final long MASK8_5 = MASKS8[5];
- private static final long MASK8_6 = MASKS8[6];
- private static final long MASK8_7 = MASKS8[7];
- private static final long MASK16_1 = MASKS16[1];
- private static final long MASK16_2 = MASKS16[2];
- private static final long MASK16_3 = MASKS16[3];
- private static final long MASK16_4 = MASKS16[4];
- private static final long MASK16_5 = MASKS16[5];
- private static final long MASK16_6 = MASKS16[6];
- private static final long MASK16_7 = MASKS16[7];
- private static final long MASK16_9 = MASKS16[9];
- private static final long MASK16_10 = MASKS16[10];
- private static final long MASK16_11 = MASKS16[11];
- private static final long MASK16_12 = MASKS16[12];
- private static final long MASK16_13 = MASKS16[13];
- private static final long MASK16_14 = MASKS16[14];
- private static final long MASK16_15 = MASKS16[15];
- private static final long MASK32_1 = MASKS32[1];
- private static final long MASK32_2 = MASKS32[2];
- private static final long MASK32_3 = MASKS32[3];
- private static final long MASK32_4 = MASKS32[4];
- private static final long MASK32_5 = MASKS32[5];
- private static final long MASK32_6 = MASKS32[6];
- private static final long MASK32_7 = MASKS32[7];
- private static final long MASK32_8 = MASKS32[8];
- private static final long MASK32_9 = MASKS32[9];
- private static final long MASK32_10 = MASKS32[10];
- private static final long MASK32_11 = MASKS32[11];
- private static final long MASK32_12 = MASKS32[12];
- private static final long MASK32_13 = MASKS32[13];
- private static final long MASK32_14 = MASKS32[14];
- private static final long MASK32_15 = MASKS32[15];
- private static final long MASK32_17 = MASKS32[17];
- private static final long MASK32_18 = MASKS32[18];
- private static final long MASK32_19 = MASKS32[19];
- private static final long MASK32_20 = MASKS32[20];
- private static final long MASK32_21 = MASKS32[21];
- private static final long MASK32_22 = MASKS32[22];
- private static final long MASK32_23 = MASKS32[23];
- private static final long MASK32_24 = MASKS32[24];
-
- /** Decode 128 integers into {@code longs}. */
- void decode(int bitsPerValue, DataInput in, long[] longs) throws IOException {
- switch (bitsPerValue) {
- case 1:
- decode1(in, tmp, longs);
- expand8(longs);
- break;
- case 2:
- decode2(in, tmp, longs);
- expand8(longs);
- break;
- case 3:
- decode3(in, tmp, longs);
- expand8(longs);
- break;
- case 4:
- decode4(in, tmp, longs);
- expand8(longs);
- break;
- case 5:
- decode5(in, tmp, longs);
- expand8(longs);
- break;
- case 6:
- decode6(in, tmp, longs);
- expand8(longs);
- break;
- case 7:
- decode7(in, tmp, longs);
- expand8(longs);
- break;
- case 8:
- decode8(in, tmp, longs);
- expand8(longs);
- break;
- case 9:
- decode9(in, tmp, longs);
- expand16(longs);
- break;
- case 10:
- decode10(in, tmp, longs);
- expand16(longs);
- break;
- case 11:
- decode11(in, tmp, longs);
- expand16(longs);
- break;
- case 12:
- decode12(in, tmp, longs);
- expand16(longs);
- break;
- case 13:
- decode13(in, tmp, longs);
- expand16(longs);
- break;
- case 14:
- decode14(in, tmp, longs);
- expand16(longs);
- break;
- case 15:
- decode15(in, tmp, longs);
- expand16(longs);
- break;
- case 16:
- decode16(in, tmp, longs);
- expand16(longs);
- break;
- case 17:
- decode17(in, tmp, longs);
- expand32(longs);
- break;
- case 18:
- decode18(in, tmp, longs);
- expand32(longs);
- break;
- case 19:
- decode19(in, tmp, longs);
- expand32(longs);
- break;
- case 20:
- decode20(in, tmp, longs);
- expand32(longs);
- break;
- case 21:
- decode21(in, tmp, longs);
- expand32(longs);
- break;
- case 22:
- decode22(in, tmp, longs);
- expand32(longs);
- break;
- case 23:
- decode23(in, tmp, longs);
- expand32(longs);
- break;
- case 24:
- decode24(in, tmp, longs);
- expand32(longs);
- break;
- default:
- decodeSlow(bitsPerValue, in, tmp, longs);
- expand32(longs);
- break;
- }
- }
-
- /** Delta-decode 128 integers into {@code longs}. */
- void decodeAndPrefixSum(int bitsPerValue, DataInput in, long base, long[] longs)
- throws IOException {
- switch (bitsPerValue) {
- case 1:
- decode1(in, tmp, longs);
- prefixSum8(longs, base);
- break;
- case 2:
- decode2(in, tmp, longs);
- prefixSum8(longs, base);
- break;
- case 3:
- decode3(in, tmp, longs);
- prefixSum8(longs, base);
- break;
- case 4:
- decode4(in, tmp, longs);
- prefixSum8(longs, base);
- break;
- case 5:
- decode5(in, tmp, longs);
- prefixSum8(longs, base);
- break;
- case 6:
- decode6(in, tmp, longs);
- prefixSum8(longs, base);
- break;
- case 7:
- decode7(in, tmp, longs);
- prefixSum8(longs, base);
- break;
- case 8:
- decode8(in, tmp, longs);
- prefixSum8(longs, base);
- break;
- case 9:
- decode9(in, tmp, longs);
- prefixSum16(longs, base);
- break;
- case 10:
- decode10(in, tmp, longs);
- prefixSum16(longs, base);
- break;
- case 11:
- decode11(in, tmp, longs);
- prefixSum16(longs, base);
- break;
- case 12:
- decode12(in, tmp, longs);
- prefixSum16(longs, base);
- break;
- case 13:
- decode13(in, tmp, longs);
- prefixSum16(longs, base);
- break;
- case 14:
- decode14(in, tmp, longs);
- prefixSum16(longs, base);
- break;
- case 15:
- decode15(in, tmp, longs);
- prefixSum16(longs, base);
- break;
- case 16:
- decode16(in, tmp, longs);
- prefixSum16(longs, base);
- break;
- case 17:
- decode17(in, tmp, longs);
- prefixSum32(longs, base);
- break;
- case 18:
- decode18(in, tmp, longs);
- prefixSum32(longs, base);
- break;
- case 19:
- decode19(in, tmp, longs);
- prefixSum32(longs, base);
- break;
- case 20:
- decode20(in, tmp, longs);
- prefixSum32(longs, base);
- break;
- case 21:
- decode21(in, tmp, longs);
- prefixSum32(longs, base);
- break;
- case 22:
- decode22(in, tmp, longs);
- prefixSum32(longs, base);
- break;
- case 23:
- decode23(in, tmp, longs);
- prefixSum32(longs, base);
- break;
- case 24:
- decode24(in, tmp, longs);
- prefixSum32(longs, base);
- break;
- default:
- decodeSlow(bitsPerValue, in, tmp, longs);
- prefixSum32(longs, base);
- break;
- }
- }
-
- private static void decode1(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, tmp, 0, 2);
- shiftLongs(tmp, 2, longs, 0, 7, MASK8_1);
- shiftLongs(tmp, 2, longs, 2, 6, MASK8_1);
- shiftLongs(tmp, 2, longs, 4, 5, MASK8_1);
- shiftLongs(tmp, 2, longs, 6, 4, MASK8_1);
- shiftLongs(tmp, 2, longs, 8, 3, MASK8_1);
- shiftLongs(tmp, 2, longs, 10, 2, MASK8_1);
- shiftLongs(tmp, 2, longs, 12, 1, MASK8_1);
- shiftLongs(tmp, 2, longs, 14, 0, MASK8_1);
- }
-
- private static void decode2(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, tmp, 0, 4);
- shiftLongs(tmp, 4, longs, 0, 6, MASK8_2);
- shiftLongs(tmp, 4, longs, 4, 4, MASK8_2);
- shiftLongs(tmp, 4, longs, 8, 2, MASK8_2);
- shiftLongs(tmp, 4, longs, 12, 0, MASK8_2);
- }
-
- private static void decode3(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, tmp, 0, 6);
- shiftLongs(tmp, 6, longs, 0, 5, MASK8_3);
- shiftLongs(tmp, 6, longs, 6, 2, MASK8_3);
- for (int iter = 0, tmpIdx = 0, longsIdx = 12; iter < 2; ++iter, tmpIdx += 3, longsIdx += 2) {
- long l0 = (tmp[tmpIdx + 0] & MASK8_2) << 1;
- l0 |= (tmp[tmpIdx + 1] >>> 1) & MASK8_1;
- longs[longsIdx + 0] = l0;
- long l1 = (tmp[tmpIdx + 1] & MASK8_1) << 2;
- l1 |= (tmp[tmpIdx + 2] & MASK8_2) << 0;
- longs[longsIdx + 1] = l1;
- }
- }
-
- private static void decode4(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, tmp, 0, 8);
- shiftLongs(tmp, 8, longs, 0, 4, MASK8_4);
- shiftLongs(tmp, 8, longs, 8, 0, MASK8_4);
- }
-
- private static void decode5(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, tmp, 0, 10);
- shiftLongs(tmp, 10, longs, 0, 3, MASK8_5);
- for (int iter = 0, tmpIdx = 0, longsIdx = 10; iter < 2; ++iter, tmpIdx += 5, longsIdx += 3) {
- long l0 = (tmp[tmpIdx + 0] & MASK8_3) << 2;
- l0 |= (tmp[tmpIdx + 1] >>> 1) & MASK8_2;
- longs[longsIdx + 0] = l0;
- long l1 = (tmp[tmpIdx + 1] & MASK8_1) << 4;
- l1 |= (tmp[tmpIdx + 2] & MASK8_3) << 1;
- l1 |= (tmp[tmpIdx + 3] >>> 2) & MASK8_1;
- longs[longsIdx + 1] = l1;
- long l2 = (tmp[tmpIdx + 3] & MASK8_2) << 3;
- l2 |= (tmp[tmpIdx + 4] & MASK8_3) << 0;
- longs[longsIdx + 2] = l2;
- }
- }
-
- private static void decode6(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, tmp, 0, 12);
- shiftLongs(tmp, 12, longs, 0, 2, MASK8_6);
- shiftLongs(tmp, 12, tmp, 0, 0, MASK8_2);
- for (int iter = 0, tmpIdx = 0, longsIdx = 12; iter < 4; ++iter, tmpIdx += 3, longsIdx += 1) {
- long l0 = tmp[tmpIdx + 0] << 4;
- l0 |= tmp[tmpIdx + 1] << 2;
- l0 |= tmp[tmpIdx + 2] << 0;
- longs[longsIdx + 0] = l0;
- }
- }
-
- private static void decode7(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, tmp, 0, 14);
- shiftLongs(tmp, 14, longs, 0, 1, MASK8_7);
- shiftLongs(tmp, 14, tmp, 0, 0, MASK8_1);
- for (int iter = 0, tmpIdx = 0, longsIdx = 14; iter < 2; ++iter, tmpIdx += 7, longsIdx += 1) {
- long l0 = tmp[tmpIdx + 0] << 6;
- l0 |= tmp[tmpIdx + 1] << 5;
- l0 |= tmp[tmpIdx + 2] << 4;
- l0 |= tmp[tmpIdx + 3] << 3;
- l0 |= tmp[tmpIdx + 4] << 2;
- l0 |= tmp[tmpIdx + 5] << 1;
- l0 |= tmp[tmpIdx + 6] << 0;
- longs[longsIdx + 0] = l0;
- }
- }
-
- private static void decode8(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, longs, 0, 16);
- }
-
- private static void decode9(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, tmp, 0, 18);
- shiftLongs(tmp, 18, longs, 0, 7, MASK16_9);
- for (int iter = 0, tmpIdx = 0, longsIdx = 18; iter < 2; ++iter, tmpIdx += 9, longsIdx += 7) {
- long l0 = (tmp[tmpIdx + 0] & MASK16_7) << 2;
- l0 |= (tmp[tmpIdx + 1] >>> 5) & MASK16_2;
- longs[longsIdx + 0] = l0;
- long l1 = (tmp[tmpIdx + 1] & MASK16_5) << 4;
- l1 |= (tmp[tmpIdx + 2] >>> 3) & MASK16_4;
- longs[longsIdx + 1] = l1;
- long l2 = (tmp[tmpIdx + 2] & MASK16_3) << 6;
- l2 |= (tmp[tmpIdx + 3] >>> 1) & MASK16_6;
- longs[longsIdx + 2] = l2;
- long l3 = (tmp[tmpIdx + 3] & MASK16_1) << 8;
- l3 |= (tmp[tmpIdx + 4] & MASK16_7) << 1;
- l3 |= (tmp[tmpIdx + 5] >>> 6) & MASK16_1;
- longs[longsIdx + 3] = l3;
- long l4 = (tmp[tmpIdx + 5] & MASK16_6) << 3;
- l4 |= (tmp[tmpIdx + 6] >>> 4) & MASK16_3;
- longs[longsIdx + 4] = l4;
- long l5 = (tmp[tmpIdx + 6] & MASK16_4) << 5;
- l5 |= (tmp[tmpIdx + 7] >>> 2) & MASK16_5;
- longs[longsIdx + 5] = l5;
- long l6 = (tmp[tmpIdx + 7] & MASK16_2) << 7;
- l6 |= (tmp[tmpIdx + 8] & MASK16_7) << 0;
- longs[longsIdx + 6] = l6;
- }
- }
-
- private static void decode10(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, tmp, 0, 20);
- shiftLongs(tmp, 20, longs, 0, 6, MASK16_10);
- for (int iter = 0, tmpIdx = 0, longsIdx = 20; iter < 4; ++iter, tmpIdx += 5, longsIdx += 3) {
- long l0 = (tmp[tmpIdx + 0] & MASK16_6) << 4;
- l0 |= (tmp[tmpIdx + 1] >>> 2) & MASK16_4;
- longs[longsIdx + 0] = l0;
- long l1 = (tmp[tmpIdx + 1] & MASK16_2) << 8;
- l1 |= (tmp[tmpIdx + 2] & MASK16_6) << 2;
- l1 |= (tmp[tmpIdx + 3] >>> 4) & MASK16_2;
- longs[longsIdx + 1] = l1;
- long l2 = (tmp[tmpIdx + 3] & MASK16_4) << 6;
- l2 |= (tmp[tmpIdx + 4] & MASK16_6) << 0;
- longs[longsIdx + 2] = l2;
- }
- }
-
- private static void decode11(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, tmp, 0, 22);
- shiftLongs(tmp, 22, longs, 0, 5, MASK16_11);
- for (int iter = 0, tmpIdx = 0, longsIdx = 22; iter < 2; ++iter, tmpIdx += 11, longsIdx += 5) {
- long l0 = (tmp[tmpIdx + 0] & MASK16_5) << 6;
- l0 |= (tmp[tmpIdx + 1] & MASK16_5) << 1;
- l0 |= (tmp[tmpIdx + 2] >>> 4) & MASK16_1;
- longs[longsIdx + 0] = l0;
- long l1 = (tmp[tmpIdx + 2] & MASK16_4) << 7;
- l1 |= (tmp[tmpIdx + 3] & MASK16_5) << 2;
- l1 |= (tmp[tmpIdx + 4] >>> 3) & MASK16_2;
- longs[longsIdx + 1] = l1;
- long l2 = (tmp[tmpIdx + 4] & MASK16_3) << 8;
- l2 |= (tmp[tmpIdx + 5] & MASK16_5) << 3;
- l2 |= (tmp[tmpIdx + 6] >>> 2) & MASK16_3;
- longs[longsIdx + 2] = l2;
- long l3 = (tmp[tmpIdx + 6] & MASK16_2) << 9;
- l3 |= (tmp[tmpIdx + 7] & MASK16_5) << 4;
- l3 |= (tmp[tmpIdx + 8] >>> 1) & MASK16_4;
- longs[longsIdx + 3] = l3;
- long l4 = (tmp[tmpIdx + 8] & MASK16_1) << 10;
- l4 |= (tmp[tmpIdx + 9] & MASK16_5) << 5;
- l4 |= (tmp[tmpIdx + 10] & MASK16_5) << 0;
- longs[longsIdx + 4] = l4;
- }
- }
-
- private static void decode12(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, tmp, 0, 24);
- shiftLongs(tmp, 24, longs, 0, 4, MASK16_12);
- shiftLongs(tmp, 24, tmp, 0, 0, MASK16_4);
- for (int iter = 0, tmpIdx = 0, longsIdx = 24; iter < 8; ++iter, tmpIdx += 3, longsIdx += 1) {
- long l0 = tmp[tmpIdx + 0] << 8;
- l0 |= tmp[tmpIdx + 1] << 4;
- l0 |= tmp[tmpIdx + 2] << 0;
- longs[longsIdx + 0] = l0;
- }
- }
-
- private static void decode13(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, tmp, 0, 26);
- shiftLongs(tmp, 26, longs, 0, 3, MASK16_13);
- for (int iter = 0, tmpIdx = 0, longsIdx = 26; iter < 2; ++iter, tmpIdx += 13, longsIdx += 3) {
- long l0 = (tmp[tmpIdx + 0] & MASK16_3) << 10;
- l0 |= (tmp[tmpIdx + 1] & MASK16_3) << 7;
- l0 |= (tmp[tmpIdx + 2] & MASK16_3) << 4;
- l0 |= (tmp[tmpIdx + 3] & MASK16_3) << 1;
- l0 |= (tmp[tmpIdx + 4] >>> 2) & MASK16_1;
- longs[longsIdx + 0] = l0;
- long l1 = (tmp[tmpIdx + 4] & MASK16_2) << 11;
- l1 |= (tmp[tmpIdx + 5] & MASK16_3) << 8;
- l1 |= (tmp[tmpIdx + 6] & MASK16_3) << 5;
- l1 |= (tmp[tmpIdx + 7] & MASK16_3) << 2;
- l1 |= (tmp[tmpIdx + 8] >>> 1) & MASK16_2;
- longs[longsIdx + 1] = l1;
- long l2 = (tmp[tmpIdx + 8] & MASK16_1) << 12;
- l2 |= (tmp[tmpIdx + 9] & MASK16_3) << 9;
- l2 |= (tmp[tmpIdx + 10] & MASK16_3) << 6;
- l2 |= (tmp[tmpIdx + 11] & MASK16_3) << 3;
- l2 |= (tmp[tmpIdx + 12] & MASK16_3) << 0;
- longs[longsIdx + 2] = l2;
- }
- }
-
- private static void decode14(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, tmp, 0, 28);
- shiftLongs(tmp, 28, longs, 0, 2, MASK16_14);
- shiftLongs(tmp, 28, tmp, 0, 0, MASK16_2);
- for (int iter = 0, tmpIdx = 0, longsIdx = 28; iter < 4; ++iter, tmpIdx += 7, longsIdx += 1) {
- long l0 = tmp[tmpIdx + 0] << 12;
- l0 |= tmp[tmpIdx + 1] << 10;
- l0 |= tmp[tmpIdx + 2] << 8;
- l0 |= tmp[tmpIdx + 3] << 6;
- l0 |= tmp[tmpIdx + 4] << 4;
- l0 |= tmp[tmpIdx + 5] << 2;
- l0 |= tmp[tmpIdx + 6] << 0;
- longs[longsIdx + 0] = l0;
- }
- }
-
- private static void decode15(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, tmp, 0, 30);
- shiftLongs(tmp, 30, longs, 0, 1, MASK16_15);
- shiftLongs(tmp, 30, tmp, 0, 0, MASK16_1);
- for (int iter = 0, tmpIdx = 0, longsIdx = 30; iter < 2; ++iter, tmpIdx += 15, longsIdx += 1) {
- long l0 = tmp[tmpIdx + 0] << 14;
- l0 |= tmp[tmpIdx + 1] << 13;
- l0 |= tmp[tmpIdx + 2] << 12;
- l0 |= tmp[tmpIdx + 3] << 11;
- l0 |= tmp[tmpIdx + 4] << 10;
- l0 |= tmp[tmpIdx + 5] << 9;
- l0 |= tmp[tmpIdx + 6] << 8;
- l0 |= tmp[tmpIdx + 7] << 7;
- l0 |= tmp[tmpIdx + 8] << 6;
- l0 |= tmp[tmpIdx + 9] << 5;
- l0 |= tmp[tmpIdx + 10] << 4;
- l0 |= tmp[tmpIdx + 11] << 3;
- l0 |= tmp[tmpIdx + 12] << 2;
- l0 |= tmp[tmpIdx + 13] << 1;
- l0 |= tmp[tmpIdx + 14] << 0;
- longs[longsIdx + 0] = l0;
- }
- }
-
- private static void decode16(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, longs, 0, 32);
- }
-
- private static void decode17(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, tmp, 0, 34);
- shiftLongs(tmp, 34, longs, 0, 15, MASK32_17);
- for (int iter = 0, tmpIdx = 0, longsIdx = 34; iter < 2; ++iter, tmpIdx += 17, longsIdx += 15) {
- long l0 = (tmp[tmpIdx + 0] & MASK32_15) << 2;
- l0 |= (tmp[tmpIdx + 1] >>> 13) & MASK32_2;
- longs[longsIdx + 0] = l0;
- long l1 = (tmp[tmpIdx + 1] & MASK32_13) << 4;
- l1 |= (tmp[tmpIdx + 2] >>> 11) & MASK32_4;
- longs[longsIdx + 1] = l1;
- long l2 = (tmp[tmpIdx + 2] & MASK32_11) << 6;
- l2 |= (tmp[tmpIdx + 3] >>> 9) & MASK32_6;
- longs[longsIdx + 2] = l2;
- long l3 = (tmp[tmpIdx + 3] & MASK32_9) << 8;
- l3 |= (tmp[tmpIdx + 4] >>> 7) & MASK32_8;
- longs[longsIdx + 3] = l3;
- long l4 = (tmp[tmpIdx + 4] & MASK32_7) << 10;
- l4 |= (tmp[tmpIdx + 5] >>> 5) & MASK32_10;
- longs[longsIdx + 4] = l4;
- long l5 = (tmp[tmpIdx + 5] & MASK32_5) << 12;
- l5 |= (tmp[tmpIdx + 6] >>> 3) & MASK32_12;
- longs[longsIdx + 5] = l5;
- long l6 = (tmp[tmpIdx + 6] & MASK32_3) << 14;
- l6 |= (tmp[tmpIdx + 7] >>> 1) & MASK32_14;
- longs[longsIdx + 6] = l6;
- long l7 = (tmp[tmpIdx + 7] & MASK32_1) << 16;
- l7 |= (tmp[tmpIdx + 8] & MASK32_15) << 1;
- l7 |= (tmp[tmpIdx + 9] >>> 14) & MASK32_1;
- longs[longsIdx + 7] = l7;
- long l8 = (tmp[tmpIdx + 9] & MASK32_14) << 3;
- l8 |= (tmp[tmpIdx + 10] >>> 12) & MASK32_3;
- longs[longsIdx + 8] = l8;
- long l9 = (tmp[tmpIdx + 10] & MASK32_12) << 5;
- l9 |= (tmp[tmpIdx + 11] >>> 10) & MASK32_5;
- longs[longsIdx + 9] = l9;
- long l10 = (tmp[tmpIdx + 11] & MASK32_10) << 7;
- l10 |= (tmp[tmpIdx + 12] >>> 8) & MASK32_7;
- longs[longsIdx + 10] = l10;
- long l11 = (tmp[tmpIdx + 12] & MASK32_8) << 9;
- l11 |= (tmp[tmpIdx + 13] >>> 6) & MASK32_9;
- longs[longsIdx + 11] = l11;
- long l12 = (tmp[tmpIdx + 13] & MASK32_6) << 11;
- l12 |= (tmp[tmpIdx + 14] >>> 4) & MASK32_11;
- longs[longsIdx + 12] = l12;
- long l13 = (tmp[tmpIdx + 14] & MASK32_4) << 13;
- l13 |= (tmp[tmpIdx + 15] >>> 2) & MASK32_13;
- longs[longsIdx + 13] = l13;
- long l14 = (tmp[tmpIdx + 15] & MASK32_2) << 15;
- l14 |= (tmp[tmpIdx + 16] & MASK32_15) << 0;
- longs[longsIdx + 14] = l14;
- }
- }
-
- private static void decode18(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, tmp, 0, 36);
- shiftLongs(tmp, 36, longs, 0, 14, MASK32_18);
- for (int iter = 0, tmpIdx = 0, longsIdx = 36; iter < 4; ++iter, tmpIdx += 9, longsIdx += 7) {
- long l0 = (tmp[tmpIdx + 0] & MASK32_14) << 4;
- l0 |= (tmp[tmpIdx + 1] >>> 10) & MASK32_4;
- longs[longsIdx + 0] = l0;
- long l1 = (tmp[tmpIdx + 1] & MASK32_10) << 8;
- l1 |= (tmp[tmpIdx + 2] >>> 6) & MASK32_8;
- longs[longsIdx + 1] = l1;
- long l2 = (tmp[tmpIdx + 2] & MASK32_6) << 12;
- l2 |= (tmp[tmpIdx + 3] >>> 2) & MASK32_12;
- longs[longsIdx + 2] = l2;
- long l3 = (tmp[tmpIdx + 3] & MASK32_2) << 16;
- l3 |= (tmp[tmpIdx + 4] & MASK32_14) << 2;
- l3 |= (tmp[tmpIdx + 5] >>> 12) & MASK32_2;
- longs[longsIdx + 3] = l3;
- long l4 = (tmp[tmpIdx + 5] & MASK32_12) << 6;
- l4 |= (tmp[tmpIdx + 6] >>> 8) & MASK32_6;
- longs[longsIdx + 4] = l4;
- long l5 = (tmp[tmpIdx + 6] & MASK32_8) << 10;
- l5 |= (tmp[tmpIdx + 7] >>> 4) & MASK32_10;
- longs[longsIdx + 5] = l5;
- long l6 = (tmp[tmpIdx + 7] & MASK32_4) << 14;
- l6 |= (tmp[tmpIdx + 8] & MASK32_14) << 0;
- longs[longsIdx + 6] = l6;
- }
- }
-
- private static void decode19(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, tmp, 0, 38);
- shiftLongs(tmp, 38, longs, 0, 13, MASK32_19);
- for (int iter = 0, tmpIdx = 0, longsIdx = 38; iter < 2; ++iter, tmpIdx += 19, longsIdx += 13) {
- long l0 = (tmp[tmpIdx + 0] & MASK32_13) << 6;
- l0 |= (tmp[tmpIdx + 1] >>> 7) & MASK32_6;
- longs[longsIdx + 0] = l0;
- long l1 = (tmp[tmpIdx + 1] & MASK32_7) << 12;
- l1 |= (tmp[tmpIdx + 2] >>> 1) & MASK32_12;
- longs[longsIdx + 1] = l1;
- long l2 = (tmp[tmpIdx + 2] & MASK32_1) << 18;
- l2 |= (tmp[tmpIdx + 3] & MASK32_13) << 5;
- l2 |= (tmp[tmpIdx + 4] >>> 8) & MASK32_5;
- longs[longsIdx + 2] = l2;
- long l3 = (tmp[tmpIdx + 4] & MASK32_8) << 11;
- l3 |= (tmp[tmpIdx + 5] >>> 2) & MASK32_11;
- longs[longsIdx + 3] = l3;
- long l4 = (tmp[tmpIdx + 5] & MASK32_2) << 17;
- l4 |= (tmp[tmpIdx + 6] & MASK32_13) << 4;
- l4 |= (tmp[tmpIdx + 7] >>> 9) & MASK32_4;
- longs[longsIdx + 4] = l4;
- long l5 = (tmp[tmpIdx + 7] & MASK32_9) << 10;
- l5 |= (tmp[tmpIdx + 8] >>> 3) & MASK32_10;
- longs[longsIdx + 5] = l5;
- long l6 = (tmp[tmpIdx + 8] & MASK32_3) << 16;
- l6 |= (tmp[tmpIdx + 9] & MASK32_13) << 3;
- l6 |= (tmp[tmpIdx + 10] >>> 10) & MASK32_3;
- longs[longsIdx + 6] = l6;
- long l7 = (tmp[tmpIdx + 10] & MASK32_10) << 9;
- l7 |= (tmp[tmpIdx + 11] >>> 4) & MASK32_9;
- longs[longsIdx + 7] = l7;
- long l8 = (tmp[tmpIdx + 11] & MASK32_4) << 15;
- l8 |= (tmp[tmpIdx + 12] & MASK32_13) << 2;
- l8 |= (tmp[tmpIdx + 13] >>> 11) & MASK32_2;
- longs[longsIdx + 8] = l8;
- long l9 = (tmp[tmpIdx + 13] & MASK32_11) << 8;
- l9 |= (tmp[tmpIdx + 14] >>> 5) & MASK32_8;
- longs[longsIdx + 9] = l9;
- long l10 = (tmp[tmpIdx + 14] & MASK32_5) << 14;
- l10 |= (tmp[tmpIdx + 15] & MASK32_13) << 1;
- l10 |= (tmp[tmpIdx + 16] >>> 12) & MASK32_1;
- longs[longsIdx + 10] = l10;
- long l11 = (tmp[tmpIdx + 16] & MASK32_12) << 7;
- l11 |= (tmp[tmpIdx + 17] >>> 6) & MASK32_7;
- longs[longsIdx + 11] = l11;
- long l12 = (tmp[tmpIdx + 17] & MASK32_6) << 13;
- l12 |= (tmp[tmpIdx + 18] & MASK32_13) << 0;
- longs[longsIdx + 12] = l12;
- }
- }
-
- private static void decode20(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, tmp, 0, 40);
- shiftLongs(tmp, 40, longs, 0, 12, MASK32_20);
- for (int iter = 0, tmpIdx = 0, longsIdx = 40; iter < 8; ++iter, tmpIdx += 5, longsIdx += 3) {
- long l0 = (tmp[tmpIdx + 0] & MASK32_12) << 8;
- l0 |= (tmp[tmpIdx + 1] >>> 4) & MASK32_8;
- longs[longsIdx + 0] = l0;
- long l1 = (tmp[tmpIdx + 1] & MASK32_4) << 16;
- l1 |= (tmp[tmpIdx + 2] & MASK32_12) << 4;
- l1 |= (tmp[tmpIdx + 3] >>> 8) & MASK32_4;
- longs[longsIdx + 1] = l1;
- long l2 = (tmp[tmpIdx + 3] & MASK32_8) << 12;
- l2 |= (tmp[tmpIdx + 4] & MASK32_12) << 0;
- longs[longsIdx + 2] = l2;
- }
- }
-
- private static void decode21(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, tmp, 0, 42);
- shiftLongs(tmp, 42, longs, 0, 11, MASK32_21);
- for (int iter = 0, tmpIdx = 0, longsIdx = 42; iter < 2; ++iter, tmpIdx += 21, longsIdx += 11) {
- long l0 = (tmp[tmpIdx + 0] & MASK32_11) << 10;
- l0 |= (tmp[tmpIdx + 1] >>> 1) & MASK32_10;
- longs[longsIdx + 0] = l0;
- long l1 = (tmp[tmpIdx + 1] & MASK32_1) << 20;
- l1 |= (tmp[tmpIdx + 2] & MASK32_11) << 9;
- l1 |= (tmp[tmpIdx + 3] >>> 2) & MASK32_9;
- longs[longsIdx + 1] = l1;
- long l2 = (tmp[tmpIdx + 3] & MASK32_2) << 19;
- l2 |= (tmp[tmpIdx + 4] & MASK32_11) << 8;
- l2 |= (tmp[tmpIdx + 5] >>> 3) & MASK32_8;
- longs[longsIdx + 2] = l2;
- long l3 = (tmp[tmpIdx + 5] & MASK32_3) << 18;
- l3 |= (tmp[tmpIdx + 6] & MASK32_11) << 7;
- l3 |= (tmp[tmpIdx + 7] >>> 4) & MASK32_7;
- longs[longsIdx + 3] = l3;
- long l4 = (tmp[tmpIdx + 7] & MASK32_4) << 17;
- l4 |= (tmp[tmpIdx + 8] & MASK32_11) << 6;
- l4 |= (tmp[tmpIdx + 9] >>> 5) & MASK32_6;
- longs[longsIdx + 4] = l4;
- long l5 = (tmp[tmpIdx + 9] & MASK32_5) << 16;
- l5 |= (tmp[tmpIdx + 10] & MASK32_11) << 5;
- l5 |= (tmp[tmpIdx + 11] >>> 6) & MASK32_5;
- longs[longsIdx + 5] = l5;
- long l6 = (tmp[tmpIdx + 11] & MASK32_6) << 15;
- l6 |= (tmp[tmpIdx + 12] & MASK32_11) << 4;
- l6 |= (tmp[tmpIdx + 13] >>> 7) & MASK32_4;
- longs[longsIdx + 6] = l6;
- long l7 = (tmp[tmpIdx + 13] & MASK32_7) << 14;
- l7 |= (tmp[tmpIdx + 14] & MASK32_11) << 3;
- l7 |= (tmp[tmpIdx + 15] >>> 8) & MASK32_3;
- longs[longsIdx + 7] = l7;
- long l8 = (tmp[tmpIdx + 15] & MASK32_8) << 13;
- l8 |= (tmp[tmpIdx + 16] & MASK32_11) << 2;
- l8 |= (tmp[tmpIdx + 17] >>> 9) & MASK32_2;
- longs[longsIdx + 8] = l8;
- long l9 = (tmp[tmpIdx + 17] & MASK32_9) << 12;
- l9 |= (tmp[tmpIdx + 18] & MASK32_11) << 1;
- l9 |= (tmp[tmpIdx + 19] >>> 10) & MASK32_1;
- longs[longsIdx + 9] = l9;
- long l10 = (tmp[tmpIdx + 19] & MASK32_10) << 11;
- l10 |= (tmp[tmpIdx + 20] & MASK32_11) << 0;
- longs[longsIdx + 10] = l10;
- }
- }
-
- private static void decode22(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, tmp, 0, 44);
- shiftLongs(tmp, 44, longs, 0, 10, MASK32_22);
- for (int iter = 0, tmpIdx = 0, longsIdx = 44; iter < 4; ++iter, tmpIdx += 11, longsIdx += 5) {
- long l0 = (tmp[tmpIdx + 0] & MASK32_10) << 12;
- l0 |= (tmp[tmpIdx + 1] & MASK32_10) << 2;
- l0 |= (tmp[tmpIdx + 2] >>> 8) & MASK32_2;
- longs[longsIdx + 0] = l0;
- long l1 = (tmp[tmpIdx + 2] & MASK32_8) << 14;
- l1 |= (tmp[tmpIdx + 3] & MASK32_10) << 4;
- l1 |= (tmp[tmpIdx + 4] >>> 6) & MASK32_4;
- longs[longsIdx + 1] = l1;
- long l2 = (tmp[tmpIdx + 4] & MASK32_6) << 16;
- l2 |= (tmp[tmpIdx + 5] & MASK32_10) << 6;
- l2 |= (tmp[tmpIdx + 6] >>> 4) & MASK32_6;
- longs[longsIdx + 2] = l2;
- long l3 = (tmp[tmpIdx + 6] & MASK32_4) << 18;
- l3 |= (tmp[tmpIdx + 7] & MASK32_10) << 8;
- l3 |= (tmp[tmpIdx + 8] >>> 2) & MASK32_8;
- longs[longsIdx + 3] = l3;
- long l4 = (tmp[tmpIdx + 8] & MASK32_2) << 20;
- l4 |= (tmp[tmpIdx + 9] & MASK32_10) << 10;
- l4 |= (tmp[tmpIdx + 10] & MASK32_10) << 0;
- longs[longsIdx + 4] = l4;
- }
- }
-
- private static void decode23(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, tmp, 0, 46);
- shiftLongs(tmp, 46, longs, 0, 9, MASK32_23);
- for (int iter = 0, tmpIdx = 0, longsIdx = 46; iter < 2; ++iter, tmpIdx += 23, longsIdx += 9) {
- long l0 = (tmp[tmpIdx + 0] & MASK32_9) << 14;
- l0 |= (tmp[tmpIdx + 1] & MASK32_9) << 5;
- l0 |= (tmp[tmpIdx + 2] >>> 4) & MASK32_5;
- longs[longsIdx + 0] = l0;
- long l1 = (tmp[tmpIdx + 2] & MASK32_4) << 19;
- l1 |= (tmp[tmpIdx + 3] & MASK32_9) << 10;
- l1 |= (tmp[tmpIdx + 4] & MASK32_9) << 1;
- l1 |= (tmp[tmpIdx + 5] >>> 8) & MASK32_1;
- longs[longsIdx + 1] = l1;
- long l2 = (tmp[tmpIdx + 5] & MASK32_8) << 15;
- l2 |= (tmp[tmpIdx + 6] & MASK32_9) << 6;
- l2 |= (tmp[tmpIdx + 7] >>> 3) & MASK32_6;
- longs[longsIdx + 2] = l2;
- long l3 = (tmp[tmpIdx + 7] & MASK32_3) << 20;
- l3 |= (tmp[tmpIdx + 8] & MASK32_9) << 11;
- l3 |= (tmp[tmpIdx + 9] & MASK32_9) << 2;
- l3 |= (tmp[tmpIdx + 10] >>> 7) & MASK32_2;
- longs[longsIdx + 3] = l3;
- long l4 = (tmp[tmpIdx + 10] & MASK32_7) << 16;
- l4 |= (tmp[tmpIdx + 11] & MASK32_9) << 7;
- l4 |= (tmp[tmpIdx + 12] >>> 2) & MASK32_7;
- longs[longsIdx + 4] = l4;
- long l5 = (tmp[tmpIdx + 12] & MASK32_2) << 21;
- l5 |= (tmp[tmpIdx + 13] & MASK32_9) << 12;
- l5 |= (tmp[tmpIdx + 14] & MASK32_9) << 3;
- l5 |= (tmp[tmpIdx + 15] >>> 6) & MASK32_3;
- longs[longsIdx + 5] = l5;
- long l6 = (tmp[tmpIdx + 15] & MASK32_6) << 17;
- l6 |= (tmp[tmpIdx + 16] & MASK32_9) << 8;
- l6 |= (tmp[tmpIdx + 17] >>> 1) & MASK32_8;
- longs[longsIdx + 6] = l6;
- long l7 = (tmp[tmpIdx + 17] & MASK32_1) << 22;
- l7 |= (tmp[tmpIdx + 18] & MASK32_9) << 13;
- l7 |= (tmp[tmpIdx + 19] & MASK32_9) << 4;
- l7 |= (tmp[tmpIdx + 20] >>> 5) & MASK32_4;
- longs[longsIdx + 7] = l7;
- long l8 = (tmp[tmpIdx + 20] & MASK32_5) << 18;
- l8 |= (tmp[tmpIdx + 21] & MASK32_9) << 9;
- l8 |= (tmp[tmpIdx + 22] & MASK32_9) << 0;
- longs[longsIdx + 8] = l8;
- }
- }
-
- private static void decode24(DataInput in, long[] tmp, long[] longs) throws IOException {
- readLELongs(in, tmp, 0, 48);
- shiftLongs(tmp, 48, longs, 0, 8, MASK32_24);
- shiftLongs(tmp, 48, tmp, 0, 0, MASK32_8);
- for (int iter = 0, tmpIdx = 0, longsIdx = 48; iter < 16; ++iter, tmpIdx += 3, longsIdx += 1) {
- long l0 = tmp[tmpIdx + 0] << 16;
- l0 |= tmp[tmpIdx + 1] << 8;
- l0 |= tmp[tmpIdx + 2] << 0;
- longs[longsIdx + 0] = l0;
- }
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84Codec.java
deleted file mode 100644
index e2f45ca5961..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84Codec.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene84;
-
-import java.util.Objects;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat.Mode;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat;
-import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;
-import org.apache.lucene.backward_codecs.lucene60.Lucene60PointsFormat;
-import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat;
-import org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.CompoundFormat;
-import org.apache.lucene.codecs.DocValuesFormat;
-import org.apache.lucene.codecs.FieldInfosFormat;
-import org.apache.lucene.codecs.FilterCodec;
-import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.LiveDocsFormat;
-import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.PointsFormat;
-import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.SegmentInfoFormat;
-import org.apache.lucene.codecs.StoredFieldsFormat;
-import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
-import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
-
-/**
- * Implements the Lucene 8.4 index format, with configurable per-field postings and docvalues
- * formats.
- *
- * <p>If you want to reuse functionality of this codec in another codec, extend {@link FilterCodec}.
- *
- * @see org.apache.lucene.backward_codecs.lucene84 package documentation for file format details.
- * @lucene.experimental
- */
-public class Lucene84Codec extends Codec {
- private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat();
- private final FieldInfosFormat fieldInfosFormat = new Lucene60FieldInfosFormat();
- private final SegmentInfoFormat segmentInfosFormat = new Lucene70SegmentInfoFormat();
- private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
- private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
- private final PostingsFormat defaultFormat;
-
- private final PostingsFormat postingsFormat =
- new PerFieldPostingsFormat() {
- @Override
- public PostingsFormat getPostingsFormatForField(String field) {
- return Lucene84Codec.this.getPostingsFormatForField(field);
- }
- };
-
- private final DocValuesFormat docValuesFormat =
- new PerFieldDocValuesFormat() {
- @Override
- public DocValuesFormat getDocValuesFormatForField(String field) {
- return Lucene84Codec.this.getDocValuesFormatForField(field);
- }
- };
-
- private final StoredFieldsFormat storedFieldsFormat;
-
- /** Instantiates a new codec. */
- public Lucene84Codec() {
- this(Mode.BEST_SPEED);
- }
-
- /**
- * Instantiates a new codec, specifying the stored fields compression mode to use.
- *
- * @param mode stored fields compression mode to use for newly flushed/merged segments.
- */
- public Lucene84Codec(Mode mode) {
- super("Lucene84");
- this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Objects.requireNonNull(mode));
- this.defaultFormat = new Lucene84PostingsFormat();
- }
-
- @Override
- public StoredFieldsFormat storedFieldsFormat() {
- return storedFieldsFormat;
- }
-
- @Override
- public TermVectorsFormat termVectorsFormat() {
- return vectorsFormat;
- }
-
- @Override
- public PostingsFormat postingsFormat() {
- return postingsFormat;
- }
-
- @Override
- public final FieldInfosFormat fieldInfosFormat() {
- return fieldInfosFormat;
- }
-
- @Override
- public SegmentInfoFormat segmentInfoFormat() {
- return segmentInfosFormat;
- }
-
- @Override
- public final LiveDocsFormat liveDocsFormat() {
- return liveDocsFormat;
- }
-
- @Override
- public CompoundFormat compoundFormat() {
- return compoundFormat;
- }
-
- @Override
- public PointsFormat pointsFormat() {
- return new Lucene60PointsFormat();
- }
-
- @Override
- public KnnVectorsFormat knnVectorsFormat() {
- return KnnVectorsFormat.EMPTY;
- }
-
- /**
- * Returns the postings format that should be used for writing new segments of <code>field</code>.
- *
- * <p>The default implementation always returns "Lucene84".
- *
- * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
- * future version of Lucene are only guaranteed to be able to read the default implementation.
- */
- public PostingsFormat getPostingsFormatForField(String field) {
- return defaultFormat;
- }
-
- /**
- * Returns the docvalues format that should be used for writing new segments of <code>field</code>
- * .
- *
- * <p>The default implementation always returns "Lucene80".
- *
- * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
- * future version of Lucene are only guaranteed to be able to read the default implementation.
- */
- public DocValuesFormat getDocValuesFormatForField(String field) {
- return defaultDVFormat;
- }
-
- @Override
- public final DocValuesFormat docValuesFormat() {
- return docValuesFormat;
- }
-
- private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene80");
-
- private final NormsFormat normsFormat = new Lucene80NormsFormat();
-
- @Override
- public NormsFormat normsFormat() {
- return normsFormat;
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84PostingsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84PostingsFormat.java
deleted file mode 100644
index 0c63d244fbc..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84PostingsFormat.java
+++ /dev/null
@@ -1,474 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene84;
-
-import java.io.IOException;
-import org.apache.lucene.backward_codecs.lucene40.blocktree.Lucene40BlockTreeTermsReader;
-import org.apache.lucene.codecs.BlockTermState;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.FieldsConsumer;
-import org.apache.lucene.codecs.FieldsProducer;
-import org.apache.lucene.codecs.MultiLevelSkipListWriter;
-import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.PostingsReaderBase;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.index.TermState;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.packed.PackedInts;
-
-/**
- * Lucene 5.0 postings format, which encodes postings in packed integer blocks for fast decode.
- *
- * <p>Basic idea:
- *
- * <ul>
- * <li><b>Packed Blocks and VInt Blocks</b>:
- * <p>In packed blocks, integers are encoded with the same bit width ({@link PackedInts packed
- * format}): the block size (i.e. number of integers inside block) is fixed (currently 128).
- * Additionally blocks that are all the same value are encoded in an optimized way.
- * <p>In VInt blocks, integers are encoded as {@link DataOutput#writeVInt VInt}: the block
- * size is variable.
- * <li><b>Block structure</b>:
- * <p>When the postings are long enough, Lucene84PostingsFormat will try to encode most
- * integer data as a packed block.
- * <p>Take a term with 259 documents as an example, the first 256 document ids are encoded as
- * two packed blocks, while the remaining 3 are encoded as one VInt block.
- * <p>Different kinds of data are always encoded separately into different packed blocks, but
- * may possibly be interleaved into the same VInt block.
- * <p>This strategy is applied to pairs: &lt;document number, frequency&gt;, &lt;position,
- * payload length&gt;, &lt;position, offset start, offset length&gt;, and &lt;position,
- * payload length, offsetstart, offset length&gt;.
- * <li><b>Skipdata settings</b>:
- * <p>The structure of skip table is quite similar to previous version of Lucene. Skip
- * interval is the same as block size, and each skip entry points to the beginning of each
- * block. However, for the first block, skip data is omitted.
- * <li><b>Positions, Payloads, and Offsets</b>:
- * <p>A position is an integer indicating where the term occurs within one document. A payload
- * is a blob of metadata associated with current position. An offset is a pair of integers
- * indicating the tokenized start/end offsets for given term in current position: it is
- * essentially a specialized payload.
- * <p>When payloads and offsets are not omitted, numPositions==numPayloads==numOffsets
- * (assuming a null payload contributes one count). As mentioned in block structure, it is
- * possible to encode these three either combined or separately.
- * <p>In all cases, payloads and offsets are stored together. When encoded as a packed block,
- * position data is separated out as .pos, while payloads and offsets are encoded in .pay
- * (payload metadata will also be stored directly in .pay). When encoded as VInt blocks, all
- * these three are stored interleaved into the .pos (so is payload metadata).
- * <p>With this strategy, the majority of payload and offset data will be outside .pos file.
- * So for queries that require only position data, running on a full index with payloads and
- * offsets, this reduces disk pre-fetches.
- * </ul>
- *
- * <p>Files and detailed format:
- *
- * <ul>
- * <li><code>.tim</code>: <a href="#Termdictionary">Term Dictionary</a>
- * <li><code>.tip</code>: <a href="#Termindex">Term Index</a>
- * <li><code>.doc</code>: <a href="#Frequencies">Frequencies and Skip Data</a>
- * <li><code>.pos</code>: <a href="#Positions">Positions</a>
- * <li><code>.pay</code>: <a href="#Payloads">Payloads and Offsets</a>
- * </ul>
- *
- * <a id="Termdictionary"></a>
- *
- * <dl>
- * <dd><b>Term Dictionary</b>
- * <p>The .tim file contains the list of terms in each field along with per-term statistics
- * (such as docfreq) and pointers to the frequencies, positions, payload and skip data in the
- * .doc, .pos, and .pay files. See {@code BlockTreeTermsWriter} for more details on the
- * format.
- * <p>NOTE: The term dictionary can plug into different postings implementations: the postings
- * writer/reader are actually responsible for encoding and decoding the PostingsHeader and
- * TermMetadata sections described here:
- * <ul>
- * <li>PostingsHeader --&gt; Header, PackedBlockSize
- * <li>TermMetadata --&gt; (DocFPDelta|SingletonDocID), PosFPDelta?, PosVIntBlockFPDelta?,
- * PayFPDelta?, SkipFPDelta?
- * <li>Header, --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
- * <li>PackedBlockSize, SingletonDocID --&gt; {@link DataOutput#writeVInt VInt}
- * <li>DocFPDelta, PosFPDelta, PayFPDelta, PosVIntBlockFPDelta, SkipFPDelta --&gt; {@link
- * DataOutput#writeVLong VLong}
- * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
- * </ul>
- * <p>Notes:
- * <ul>
- * <li>Header is a {@link CodecUtil#writeIndexHeader IndexHeader} storing the version
- * information for the postings.
- * <li>PackedBlockSize is the fixed block size for packed blocks. In packed block, bit width
- * is determined by the largest integer. Smaller block size result in smaller variance
- * among width of integers hence smaller indexes. Larger block size result in more
- * efficient bulk i/o hence better acceleration. This value should always be a multiple
- * of 64, currently fixed as 128 as a tradeoff. It is also the skip interval used to
- * accelerate {@link org.apache.lucene.index.PostingsEnum#advance(int)}.
- * <li>DocFPDelta determines the position of this term's TermFreqs within the .doc file. In
- * particular, it is the difference of file offset between this term's data and previous
- * term's data (or zero, for the first term in the block).On disk it is stored as the
- * difference from previous value in sequence.
- * <li>PosFPDelta determines the position of this term's TermPositions within the .pos file.
- * While PayFPDelta determines the position of this term's &lt;TermPayloads,
- * TermOffsets?&gt; within the .pay file. Similar to DocFPDelta, it is the difference
- * between two file positions (or neglected, for fields that omit payloads and offsets).
- * <li>PosVIntBlockFPDelta determines the position of this term's last TermPosition in last
- * pos packed block within the .pos file. It is synonym for PayVIntBlockFPDelta or
- * OffsetVIntBlockFPDelta. This is actually used to indicate whether it is necessary to
- * load following payloads and offsets from .pos instead of .pay. Every time a new block
- * of positions are to be loaded, the PostingsReader will use this value to check
- * whether current block is packed format or VInt. When packed format, payloads and
- * offsets are fetched from .pay, otherwise from .pos. (this value is neglected when
- * total number of positions i.e. totalTermFreq is less or equal to PackedBlockSize).
- * <li>SkipFPDelta determines the position of this term's SkipData within the .doc file. In
- * particular, it is the length of the TermFreq data. SkipDelta is only stored if
- * DocFreq is not smaller than SkipMinimum (i.e. 128 in Lucene84PostingsFormat).
- * <li>SingletonDocID is an optimization when a term only appears in one document. In this
- * case, instead of writing a file pointer to the .doc file (DocFPDelta), and then a
- * VIntBlock at that location, the single document ID is written to the term dictionary.
- * </ul>
- * </dl>
- *
- * <a id="Termindex"></a>
- *
- * <dl>
- * <dd><b>Term Index</b>
- * <p>The .tip file contains an index into the term dictionary, so that it can be accessed
- * randomly. See {@code BlockTreeTermsWriter} for more details on the format.
- * </dl>
- *
- * <a id="Frequencies"></a>
- *
- * <dl>
- * <dd><b>Frequencies and Skip Data</b>
- * <p>The .doc file contains the lists of documents which contain each term, along with the
- * frequency of the term in that document (except when frequencies are omitted: {@link
- * IndexOptions#DOCS}). It also saves skip data to the beginning of each packed or VInt block,
- * when the length of document list is larger than packed block size.
- * <ul>
- * <li>docFile(.doc) --&gt; Header, &lt;TermFreqs, SkipData?&gt;<sup>TermCount</sup>, Footer
- * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
- * <li>TermFreqs --&gt; &lt;PackedBlock&gt; <sup>PackedDocBlockNum</sup>, VIntBlock?
- * <li>PackedBlock --&gt; PackedDocDeltaBlock, PackedFreqBlock?
- * <li>VIntBlock --&gt; &lt;DocDelta[,
- * Freq?]&gt;<sup>DocFreq-PackedBlockSize*PackedDocBlockNum</sup>
- * <li>SkipData --&gt; &lt;&lt;SkipLevelLength, SkipLevel&gt; <sup>NumSkipLevels-1</sup>,
- * SkipLevel&gt;, SkipDatum?
- * <li>SkipLevel --&gt; &lt;SkipDatum&gt; <sup>TrimmedDocFreq/(PackedBlockSize^(Level +
- * 1))</sup>
- * <li>SkipDatum --&gt; DocSkip, DocFPSkip, &lt;PosFPSkip, PosBlockOffset, PayLength?,
- * PayFPSkip?&gt;?, SkipChildLevelPointer?
- * <li>PackedDocDeltaBlock, PackedFreqBlock --&gt; {@link PackedInts PackedInts}
- * <li>DocDelta, Freq, DocSkip, DocFPSkip, PosFPSkip, PosBlockOffset, PayByteUpto, PayFPSkip
- * --&gt; {@link DataOutput#writeVInt VInt}
- * <li>SkipChildLevelPointer --&gt; {@link DataOutput#writeVLong VLong}
- * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
- * </ul>
- * <p>Notes:
- * <ul>
- * <li>PackedDocDeltaBlock is theoretically generated from two steps:
- * <ol>
- * <li>Calculate the difference between each document number and previous one, and get
- * a d-gaps list (for the first document, use absolute value);
- * <li>For those d-gaps from first one to
- * PackedDocBlockNum*PackedBlockSize<sup>th</sup>, separately encode as packed
- * blocks.
- * </ol>
- * If frequencies are not omitted, PackedFreqBlock will be generated without d-gap step.
- * <li>VIntBlock stores remaining d-gaps (along with frequencies when possible) with a
- * format that encodes DocDelta and Freq:
- * <p>DocDelta: if frequencies are indexed, this determines both the document number and
- * the frequency. In particular, DocDelta/2 is the difference between this document
- * number and the previous document number (or zero when this is the first document in a
- * TermFreqs). When DocDelta is odd, the frequency is one. When DocDelta is even, the
- * frequency is read as another VInt. If frequencies are omitted, DocDelta contains the
- * gap (not multiplied by 2) between document numbers and no frequency information is
- * stored.
- * <p>For example, the TermFreqs for a term which occurs once in document seven and
- * three times in document eleven, with frequencies indexed, would be the following
- * sequence of VInts:
- * <p>15, 8, 3
- * <p>If frequencies were omitted ({@link IndexOptions#DOCS}) it would be this sequence
- * of VInts instead:
- * <p>7,4
- * <li>PackedDocBlockNum is the number of packed blocks for current term's docids or
- * frequencies. In particular, PackedDocBlockNum = floor(DocFreq/PackedBlockSize)
- * <li>TrimmedDocFreq = DocFreq % PackedBlockSize == 0 ? DocFreq - 1 : DocFreq. We use this
- * trick since the definition of skip entry is a little different from base interface.
- * In {@link MultiLevelSkipListWriter}, skip data is assumed to be saved for
- * skipInterval<sup>th</sup>, 2*skipInterval<sup>th</sup> ... posting in the list.
- * However, in Lucene84PostingsFormat, the skip data is saved for
- * skipInterval+1<sup>th</sup>, 2*skipInterval+1<sup>th</sup> ... posting
- * (skipInterval==PackedBlockSize in this case). When DocFreq is multiple of
- * PackedBlockSize, MultiLevelSkipListWriter will expect one more skip data than
- * Lucene84SkipWriter.
- * <li>SkipDatum is the metadata of one skip entry. For the first block (no matter packed or
- * VInt), it is omitted.
- * <li>DocSkip records the document number of every PackedBlockSize<sup>th</sup> document
- * number in the postings (i.e. last document number in each packed block). On disk it
- * is stored as the difference from previous value in the sequence.
- * <li>DocFPSkip records the file offsets of each block (excluding )posting at
- * PackedBlockSize+1<sup>th</sup>, 2*PackedBlockSize+1<sup>th</sup> ... , in DocFile.
- * The file offsets are relative to the start of current term's TermFreqs. On disk it is
- * also stored as the difference from previous SkipDatum in the sequence.
- * <li>Since positions and payloads are also block encoded, the skip should skip to related
- * block first, then fetch the values according to in-block offset. PosFPSkip and
- * PayFPSkip record the file offsets of related block in .pos and .pay, respectively.
- * While PosBlockOffset indicates which value to fetch inside the related block
- * (PayBlockOffset is unnecessary since it is always equal to PosBlockOffset). Same as
- * DocFPSkip, the file offsets are relative to the start of current term's TermFreqs,
- * and stored as a difference sequence.
- * <li>PayByteUpto indicates the start offset of the current payload. It is equivalent to
- * the sum of the payload lengths in the current block up to PosBlockOffset
- * </ul>
- * </dl>
- *
- * <a id="Positions"></a>
- *
- * <dl>
- * <dd><b>Positions</b>
- * <p>The .pos file contains the lists of positions that each term occurs at within documents.
- * It also sometimes stores part of payloads and offsets for speedup.
- * <ul>
- * <li>PosFile(.pos) --&gt; Header, &lt;TermPositions&gt; <sup>TermCount</sup>, Footer
- * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
- * <li>TermPositions --&gt; &lt;PackedPosDeltaBlock&gt; <sup>PackedPosBlockNum</sup>,
- * VIntBlock?
- * <li>VIntBlock --&gt; &lt;PositionDelta[, PayloadLength?], PayloadData?, OffsetDelta?,
- * OffsetLength?&gt;<sup>PosVIntCount</sup>
- * <li>PackedPosDeltaBlock --&gt; {@link PackedInts PackedInts}
- * <li>PositionDelta, OffsetDelta, OffsetLength --&gt; {@link DataOutput#writeVInt VInt}
- * <li>PayloadData --&gt; {@link DataOutput#writeByte byte}<sup>PayLength</sup>
- * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
- * </ul>
- * <p>Notes:
- * <ul>
- * <li>TermPositions are order by term (terms are implicit, from the term dictionary), and
- * position values for each term document pair are incremental, and ordered by document
- * number.
- * <li>PackedPosBlockNum is the number of packed blocks for current term's positions,
- * payloads or offsets. In particular, PackedPosBlockNum =
- * floor(totalTermFreq/PackedBlockSize)
- * <li>PosVIntCount is the number of positions encoded as VInt format. In particular,
- * PosVIntCount = totalTermFreq - PackedPosBlockNum*PackedBlockSize
- * <li>The procedure how PackedPosDeltaBlock is generated is the same as PackedDocDeltaBlock
- * in chapter <a href="#Frequencies">Frequencies and Skip Data</a>.
- * <li>PositionDelta is, if payloads are disabled for the term's field, the difference
- * between the position of the current occurrence in the document and the previous
- * occurrence (or zero, if this is the first occurrence in this document). If payloads
- * are enabled for the term's field, then PositionDelta/2 is the difference between the
- * current and the previous position. If payloads are enabled and PositionDelta is odd,
- * then PayloadLength is stored, indicating the length of the payload at the current
- * term position.
- * <li>For example, the TermPositions for a term which occurs as the fourth term in one
- * document, and as the fifth and ninth term in a subsequent document, would be the
- * following sequence of VInts (payloads disabled):
- * <p>4, 5, 4
- * <li>PayloadData is metadata associated with the current term position. If PayloadLength
- * is stored at the current position, then it indicates the length of this payload. If
- * PayloadLength is not stored, then this payload has the same length as the payload at
- * the previous position.
- * <li>OffsetDelta/2 is the difference between this position's startOffset from the previous
- * occurrence (or zero, if this is the first occurrence in this document). If
- * OffsetDelta is odd, then the length (endOffset-startOffset) differs from the previous
- * occurrence and an OffsetLength follows. Offset data is only written for {@link
- * IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}.
- * </ul>
- * </dl>
- *
- * <a id="Payloads"></a>
- *
- * <dl>
- * <dd><b>Payloads and Offsets</b>
- * <p>The .pay file will store payloads and offsets associated with certain term-document
- * positions. Some payloads and offsets will be separated out into .pos file, for performance
- * reasons.
- * <ul>
- * <li>PayFile(.pay): --&gt; Header, &lt;TermPayloads, TermOffsets?&gt;
- * <sup>TermCount</sup>, Footer
- * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
- * <li>TermPayloads --&gt; &lt;PackedPayLengthBlock, SumPayLength, PayData&gt;
- * <sup>PackedPayBlockNum</sup>
- * <li>TermOffsets --&gt; &lt;PackedOffsetStartDeltaBlock, PackedOffsetLengthBlock&gt;
- * <sup>PackedPayBlockNum</sup>
- * <li>PackedPayLengthBlock, PackedOffsetStartDeltaBlock, PackedOffsetLengthBlock --&gt;
- * {@link PackedInts PackedInts}
- * <li>SumPayLength --&gt; {@link DataOutput#writeVInt VInt}
- * <li>PayData --&gt; {@link DataOutput#writeByte byte}<sup>SumPayLength</sup>
- * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
- * </ul>
- * <p>Notes:
- * <ul>
- * <li>The order of TermPayloads/TermOffsets will be the same as TermPositions, note that
- * part of payload/offsets are stored in .pos.
- * <li>The procedure how PackedPayLengthBlock and PackedOffsetLengthBlock are generated is
- * the same as PackedFreqBlock in chapter <a href="#Frequencies">Frequencies and Skip
- * Data</a>. While PackedStartDeltaBlock follows a same procedure as
- * PackedDocDeltaBlock.
- * <li>PackedPayBlockNum is always equal to PackedPosBlockNum, for the same term. It is also
- * synonym for PackedOffsetBlockNum.
- * <li>SumPayLength is the total length of payloads written within one block, should be the
- * sum of PayLengths in one packed block.
- * <li>PayLength in PackedPayLengthBlock is the length of each payload associated with the
- * current position.
- * </ul>
- * </dl>
- *
- * @lucene.experimental
- */
-public class Lucene84PostingsFormat extends PostingsFormat {
-
- /**
- * Filename extension for document number, frequencies, and skip data. See chapter: <a
- * href="#Frequencies">Frequencies and Skip Data</a>
- */
- public static final String DOC_EXTENSION = "doc";
-
- /** Filename extension for positions. See chapter: <a href="#Positions">Positions</a> */
- public static final String POS_EXTENSION = "pos";
-
- /**
- * Filename extension for payloads and offsets. See chapter: <a href="#Payloads">Payloads and
- * Offsets</a>
- */
- public static final String PAY_EXTENSION = "pay";
-
- /** Size of blocks. */
- public static final int BLOCK_SIZE = ForUtil.BLOCK_SIZE;
-
- /**
- * Expert: The maximum number of skip levels. Smaller values result in slightly smaller indexes,
- * but slower skipping in big posting lists.
- */
- static final int MAX_SKIP_LEVELS = 10;
-
- static final String TERMS_CODEC = "Lucene84PostingsWriterTerms";
- static final String DOC_CODEC = "Lucene84PostingsWriterDoc";
- static final String POS_CODEC = "Lucene84PostingsWriterPos";
- static final String PAY_CODEC = "Lucene84PostingsWriterPay";
-
- // Increment version to change it
- static final int VERSION_START = 0;
- // Better compression of the terms dictionary in case most terms have a docFreq of 1
- static final int VERSION_COMPRESSED_TERMS_DICT_IDS = 1;
- static final int VERSION_CURRENT = VERSION_COMPRESSED_TERMS_DICT_IDS;
-
- /** Creates {@code Lucene84PostingsFormat} with default settings. */
- public Lucene84PostingsFormat() {
- super("Lucene84");
- }
-
- @Override
- public String toString() {
- return getName();
- }
-
- @Override
- public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- throw new UnsupportedOperationException("Old codecs may only be used for reading");
- }
-
- @Override
- public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new Lucene84PostingsReader(state);
- boolean success = false;
- try {
- FieldsProducer ret = new Lucene40BlockTreeTermsReader(postingsReader, state);
- success = true;
- return ret;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(postingsReader);
- }
- }
- }
-
- /**
- * Holds all state required for {@link Lucene84PostingsReader} to produce a {@link
- * org.apache.lucene.index.PostingsEnum} without re-seeking the terms dict.
- *
- * @lucene.internal
- */
- public static final class IntBlockTermState extends BlockTermState {
- /** file pointer to the start of the doc ids enumeration, in {@link #DOC_EXTENSION} file */
- public long docStartFP;
-
- /** file pointer to the start of the positions enumeration, in {@link #POS_EXTENSION} file */
- public long posStartFP;
-
- /** file pointer to the start of the payloads enumeration, in {@link #PAY_EXTENSION} file */
- public long payStartFP;
-
- /**
- * file offset for the start of the skip list, relative to docStartFP, if there are more than
- * {@link ForUtil#BLOCK_SIZE} docs; otherwise -1
- */
- public long skipOffset;
-
- /**
- * file offset for the last position in the last block, if there are more than {@link
- * ForUtil#BLOCK_SIZE} positions; otherwise -1
- */
- public long lastPosBlockOffset;
-
- /**
- * docid when there is a single pulsed posting, otherwise -1. freq is always implicitly
- * totalTermFreq in this case.
- */
- public int singletonDocID;
-
- /** Sole constructor. */
- public IntBlockTermState() {
- skipOffset = -1;
- lastPosBlockOffset = -1;
- singletonDocID = -1;
- }
-
- @Override
- public IntBlockTermState clone() {
- IntBlockTermState other = new IntBlockTermState();
- other.copyFrom(this);
- return other;
- }
-
- @Override
- public void copyFrom(TermState _other) {
- super.copyFrom(_other);
- IntBlockTermState other = (IntBlockTermState) _other;
- docStartFP = other.docStartFP;
- posStartFP = other.posStartFP;
- payStartFP = other.payStartFP;
- lastPosBlockOffset = other.lastPosBlockOffset;
- skipOffset = other.skipOffset;
- singletonDocID = other.singletonDocID;
- }
-
- @Override
- public String toString() {
- return super.toString()
- + " docStartFP="
- + docStartFP
- + " posStartFP="
- + posStartFP
- + " payStartFP="
- + payStartFP
- + " lastPosBlockOffset="
- + lastPosBlockOffset
- + " singletonDocID="
- + singletonDocID;
- }
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84PostingsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84PostingsReader.java
deleted file mode 100644
index 9709c7aa9d7..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84PostingsReader.java
+++ /dev/null
@@ -1,2092 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene84;
-
-import static org.apache.lucene.backward_codecs.lucene84.ForUtil.BLOCK_SIZE;
-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.DOC_CODEC;
-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.MAX_SKIP_LEVELS;
-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.PAY_CODEC;
-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.POS_CODEC;
-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.TERMS_CODEC;
-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.VERSION_COMPRESSED_TERMS_DICT_IDS;
-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.VERSION_CURRENT;
-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.VERSION_START;
-
-import java.io.IOException;
-import java.util.Arrays;
-import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.IntBlockTermState;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.BlockTermState;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.PostingsReaderBase;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.Impacts;
-import org.apache.lucene.index.ImpactsEnum;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.index.SlowImpactsEnum;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BitUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
-
-/**
- * Concrete class that reads docId(maybe frq,pos,offset,payloads) list with postings format.
- *
- * @lucene.experimental
- */
-public final class Lucene84PostingsReader extends PostingsReaderBase {
-
- private final IndexInput docIn;
- private final IndexInput posIn;
- private final IndexInput payIn;
-
- private final int version;
-
- /** Sole constructor. */
- public Lucene84PostingsReader(SegmentReadState state) throws IOException {
- boolean success = false;
- IndexInput docIn = null;
- IndexInput posIn = null;
- IndexInput payIn = null;
-
- // NOTE: these data files are too costly to verify checksum against all the bytes on open,
- // but for now we at least verify proper structure of the checksum footer: which looks
- // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
- // such as file truncation.
-
- String docName =
- IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, Lucene84PostingsFormat.DOC_EXTENSION);
- try {
- docIn = EndiannessReverserUtil.openInput(state.directory, docName, state.context);
- version =
- CodecUtil.checkIndexHeader(
- docIn,
- DOC_CODEC,
- VERSION_START,
- VERSION_CURRENT,
- state.segmentInfo.getId(),
- state.segmentSuffix);
- CodecUtil.retrieveChecksum(docIn);
-
- if (state.fieldInfos.hasProx()) {
- String proxName =
- IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, Lucene84PostingsFormat.POS_EXTENSION);
- posIn = EndiannessReverserUtil.openInput(state.directory, proxName, state.context);
- CodecUtil.checkIndexHeader(
- posIn, POS_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);
- CodecUtil.retrieveChecksum(posIn);
-
- if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {
- String payName =
- IndexFileNames.segmentFileName(
- state.segmentInfo.name,
- state.segmentSuffix,
- Lucene84PostingsFormat.PAY_EXTENSION);
- payIn = EndiannessReverserUtil.openInput(state.directory, payName, state.context);
- CodecUtil.checkIndexHeader(
- payIn, PAY_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);
- CodecUtil.retrieveChecksum(payIn);
- }
- }
-
- this.docIn = docIn;
- this.posIn = posIn;
- this.payIn = payIn;
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(docIn, posIn, payIn);
- }
- }
- }
-
- @Override
- public void init(IndexInput termsIn, SegmentReadState state) throws IOException {
- // Make sure we are talking to the matching postings writer
- CodecUtil.checkIndexHeader(
- termsIn,
- TERMS_CODEC,
- VERSION_START,
- VERSION_CURRENT,
- state.segmentInfo.getId(),
- state.segmentSuffix);
- final int indexBlockSize = termsIn.readVInt();
- if (indexBlockSize != ForUtil.BLOCK_SIZE) {
- throw new IllegalStateException(
- "index-time BLOCK_SIZE ("
- + indexBlockSize
- + ") != read-time BLOCK_SIZE ("
- + ForUtil.BLOCK_SIZE
- + ")");
- }
- }
-
- /** Read values that have been written using variable-length encoding instead of bit-packing. */
- static void readVIntBlock(
- IndexInput docIn, long[] docBuffer, long[] freqBuffer, int num, boolean indexHasFreq)
- throws IOException {
- if (indexHasFreq) {
- for (int i = 0; i < num; i++) {
- final int code = docIn.readVInt();
- docBuffer[i] = code >>> 1;
- if ((code & 1) != 0) {
- freqBuffer[i] = 1;
- } else {
- freqBuffer[i] = docIn.readVInt();
- }
- }
- } else {
- for (int i = 0; i < num; i++) {
- docBuffer[i] = docIn.readVInt();
- }
- }
- }
-
- static void prefixSum(long[] buffer, int count, long base) {
- buffer[0] += base;
- for (int i = 1; i < count; ++i) {
- buffer[i] += buffer[i - 1];
- }
- }
-
- static int findFirstGreater(long[] buffer, int target, int from) {
- for (int i = from; i < BLOCK_SIZE; ++i) {
- if (buffer[i] >= target) {
- return i;
- }
- }
- return BLOCK_SIZE;
- }
-
- @Override
- public BlockTermState newTermState() {
- return new IntBlockTermState();
- }
-
- @Override
- public void close() throws IOException {
- IOUtils.close(docIn, posIn, payIn);
- }
-
- @Override
- public void decodeTerm(
- DataInput in, FieldInfo fieldInfo, BlockTermState _termState, boolean absolute)
- throws IOException {
- final IntBlockTermState termState = (IntBlockTermState) _termState;
- final boolean fieldHasPositions =
- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- final boolean fieldHasOffsets =
- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
- >= 0;
- final boolean fieldHasPayloads = fieldInfo.hasPayloads();
-
- if (absolute) {
- termState.docStartFP = 0;
- termState.posStartFP = 0;
- termState.payStartFP = 0;
- }
-
- if (version >= VERSION_COMPRESSED_TERMS_DICT_IDS) {
- final long l = in.readVLong();
- if ((l & 0x01) == 0) {
- termState.docStartFP += l >>> 1;
- if (termState.docFreq == 1) {
- termState.singletonDocID = in.readVInt();
- } else {
- termState.singletonDocID = -1;
- }
- } else {
- assert absolute == false;
- assert termState.singletonDocID != -1;
- termState.singletonDocID += BitUtil.zigZagDecode(l >>> 1);
- }
- } else {
- termState.docStartFP += in.readVLong();
- }
-
- if (fieldHasPositions) {
- termState.posStartFP += in.readVLong();
- if (fieldHasOffsets || fieldHasPayloads) {
- termState.payStartFP += in.readVLong();
- }
- }
- if (version < VERSION_COMPRESSED_TERMS_DICT_IDS) {
- if (termState.docFreq == 1) {
- termState.singletonDocID = in.readVInt();
- } else {
- termState.singletonDocID = -1;
- }
- }
- if (fieldHasPositions) {
- if (termState.totalTermFreq > BLOCK_SIZE) {
- termState.lastPosBlockOffset = in.readVLong();
- } else {
- termState.lastPosBlockOffset = -1;
- }
- }
- if (termState.docFreq > BLOCK_SIZE) {
- termState.skipOffset = in.readVLong();
- } else {
- termState.skipOffset = -1;
- }
- }
-
- @Override
- public PostingsEnum postings(
- FieldInfo fieldInfo, BlockTermState termState, PostingsEnum reuse, int flags)
- throws IOException {
-
- boolean indexHasPositions =
- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
-
- if (indexHasPositions == false
- || PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS) == false) {
- BlockDocsEnum docsEnum;
- if (reuse instanceof BlockDocsEnum) {
- docsEnum = (BlockDocsEnum) reuse;
- if (!docsEnum.canReuse(docIn, fieldInfo)) {
- docsEnum = new BlockDocsEnum(fieldInfo);
- }
- } else {
- docsEnum = new BlockDocsEnum(fieldInfo);
- }
- return docsEnum.reset((IntBlockTermState) termState, flags);
- } else {
- EverythingEnum everythingEnum;
- if (reuse instanceof EverythingEnum) {
- everythingEnum = (EverythingEnum) reuse;
- if (!everythingEnum.canReuse(docIn, fieldInfo)) {
- everythingEnum = new EverythingEnum(fieldInfo);
- }
- } else {
- everythingEnum = new EverythingEnum(fieldInfo);
- }
- return everythingEnum.reset((IntBlockTermState) termState, flags);
- }
- }
-
- @Override
- public ImpactsEnum impacts(FieldInfo fieldInfo, BlockTermState state, int flags)
- throws IOException {
- if (state.docFreq <= BLOCK_SIZE) {
- // no skip data
- return new SlowImpactsEnum(postings(fieldInfo, state, null, flags));
- }
-
- final boolean indexHasPositions =
- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- final boolean indexHasOffsets =
- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
- >= 0;
- final boolean indexHasPayloads = fieldInfo.hasPayloads();
-
- if (indexHasPositions == false
- || PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS) == false) {
- return new BlockImpactsDocsEnum(fieldInfo, (IntBlockTermState) state);
- }
-
- if (indexHasPositions
- && PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)
- && (indexHasOffsets == false
- || PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS) == false)
- && (indexHasPayloads == false
- || PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS) == false)) {
- return new BlockImpactsPostingsEnum(fieldInfo, (IntBlockTermState) state);
- }
-
- return new BlockImpactsEverythingEnum(fieldInfo, (IntBlockTermState) state, flags);
- }
-
- final class BlockDocsEnum extends PostingsEnum {
-
- final ForUtil forUtil = new ForUtil();
- final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);
- final PForUtil pforUtil = new PForUtil(forUtil);
-
- private final long[] docBuffer = new long[BLOCK_SIZE + 1];
- private final long[] freqBuffer = new long[BLOCK_SIZE];
-
- private int docBufferUpto;
-
- private Lucene84SkipReader skipper;
- private boolean skipped;
-
- final IndexInput startDocIn;
-
- IndexInput docIn;
- final boolean indexHasFreq;
- final boolean indexHasPos;
- final boolean indexHasOffsets;
- final boolean indexHasPayloads;
-
- private int docFreq; // number of docs in this posting list
- private long totalTermFreq; // sum of freqBuffer in this posting list (or docFreq when omitted)
- private int blockUpto; // number of docs in or before the current block
- private int doc; // doc we last read
- private long accum; // accumulator for doc deltas
-
- // Where this term's postings start in the .doc file:
- private long docTermStartFP;
-
- // Where this term's skip data starts (after
- // docTermStartFP) in the .doc file (or -1 if there is
- // no skip data for this term):
- private long skipOffset;
-
- // docID for next skip point, we won't use skipper if
- // target docID is not larger than this
- private int nextSkipDoc;
-
- private boolean needsFreq; // true if the caller actually needs frequencies
- // as we read freqBuffer lazily, isFreqsRead shows if freqBuffer are read for the current block
- // always true when we don't have freqBuffer (indexHasFreq=false) or don't need freqBuffer
- // (needsFreq=false)
- private boolean isFreqsRead;
- private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1
-
- public BlockDocsEnum(FieldInfo fieldInfo) throws IOException {
- this.startDocIn = Lucene84PostingsReader.this.docIn;
- this.docIn = null;
- indexHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
- indexHasPos =
- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- indexHasOffsets =
- fieldInfo
- .getIndexOptions()
- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
- >= 0;
- indexHasPayloads = fieldInfo.hasPayloads();
- // We set the last element of docBuffer to NO_MORE_DOCS, it helps save conditionals in
- // advance()
- docBuffer[BLOCK_SIZE] = NO_MORE_DOCS;
- }
-
- public boolean canReuse(IndexInput docIn, FieldInfo fieldInfo) {
- return docIn == startDocIn
- && indexHasFreq
- == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0)
- && indexHasPos
- == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
- >= 0)
- && indexHasPayloads == fieldInfo.hasPayloads();
- }
-
- public PostingsEnum reset(IntBlockTermState termState, int flags) throws IOException {
- docFreq = termState.docFreq;
- totalTermFreq = indexHasFreq ? termState.totalTermFreq : docFreq;
- docTermStartFP = termState.docStartFP;
- skipOffset = termState.skipOffset;
- singletonDocID = termState.singletonDocID;
- if (docFreq > 1) {
- if (docIn == null) {
- // lazy init
- docIn = startDocIn.clone();
- }
- docIn.seek(docTermStartFP);
- }
-
- doc = -1;
- this.needsFreq = PostingsEnum.featureRequested(flags, PostingsEnum.FREQS);
- this.isFreqsRead = true;
- if (indexHasFreq == false || needsFreq == false) {
- for (int i = 0; i < BLOCK_SIZE; ++i) {
- freqBuffer[i] = 1;
- }
- }
- accum = 0;
- blockUpto = 0;
- nextSkipDoc = BLOCK_SIZE - 1; // we won't skip if target is found in first block
- docBufferUpto = BLOCK_SIZE;
- skipped = false;
- return this;
- }
-
- @Override
- public int freq() throws IOException {
- if (isFreqsRead == false) {
- pforUtil.decode(docIn, freqBuffer); // read freqBuffer for this block
- isFreqsRead = true;
- }
- return (int) freqBuffer[docBufferUpto - 1];
- }
-
- @Override
- public int nextPosition() throws IOException {
- return -1;
- }
-
- @Override
- public int startOffset() throws IOException {
- return -1;
- }
-
- @Override
- public int endOffset() throws IOException {
- return -1;
- }
-
- @Override
- public BytesRef getPayload() throws IOException {
- return null;
- }
-
- @Override
- public int docID() {
- return doc;
- }
-
- private void refillDocs() throws IOException {
- // Check if we skipped reading the previous block of freqBuffer, and if yes, position docIn
- // after it
- if (isFreqsRead == false) {
- pforUtil.skip(docIn);
- isFreqsRead = true;
- }
-
- final int left = docFreq - blockUpto;
- assert left >= 0;
-
- if (left >= BLOCK_SIZE) {
- forDeltaUtil.decodeAndPrefixSum(docIn, accum, docBuffer);
-
- if (indexHasFreq) {
- if (needsFreq) {
- isFreqsRead = false;
- } else {
- pforUtil.skip(docIn); // skip over freqBuffer if we don't need them at all
- }
- }
- blockUpto += BLOCK_SIZE;
- } else if (docFreq == 1) {
- docBuffer[0] = singletonDocID;
- freqBuffer[0] = totalTermFreq;
- docBuffer[1] = NO_MORE_DOCS;
- blockUpto++;
- } else {
- // Read vInts:
- readVIntBlock(docIn, docBuffer, freqBuffer, left, indexHasFreq);
- prefixSum(docBuffer, left, accum);
- docBuffer[left] = NO_MORE_DOCS;
- blockUpto += left;
- }
- accum = docBuffer[BLOCK_SIZE - 1];
- docBufferUpto = 0;
- assert docBuffer[BLOCK_SIZE] == NO_MORE_DOCS;
- }
-
- @Override
- public int nextDoc() throws IOException {
- if (docBufferUpto == BLOCK_SIZE) {
- refillDocs(); // we don't need to load freqBuffer for now (will be loaded later if
- // necessary)
- }
-
- doc = (int) docBuffer[docBufferUpto];
- docBufferUpto++;
- return doc;
- }
-
- @Override
- public int advance(int target) throws IOException {
- // current skip docID < docIDs generated from current buffer <= next skip docID
- // we don't need to skip if target is buffered already
- if (docFreq > BLOCK_SIZE && target > nextSkipDoc) {
-
- if (skipper == null) {
- // Lazy init: first time this enum has ever been used for skipping
- skipper =
- new Lucene84SkipReader(
- docIn.clone(), MAX_SKIP_LEVELS, indexHasPos, indexHasOffsets, indexHasPayloads);
- }
-
- if (!skipped) {
- assert skipOffset != -1;
- // This is the first time this enum has skipped
- // since reset() was called; load the skip data:
- skipper.init(docTermStartFP + skipOffset, docTermStartFP, 0, 0, docFreq);
- skipped = true;
- }
-
- // always plus one to fix the result, since skip position in Lucene84SkipReader
- // is a little different from MultiLevelSkipListReader
- final int newDocUpto = skipper.skipTo(target) + 1;
-
- if (newDocUpto >= blockUpto) {
- // Skipper moved
- assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;
- blockUpto = newDocUpto;
-
- // Force to read next block
- docBufferUpto = BLOCK_SIZE;
- accum = skipper.getDoc(); // actually, this is just lastSkipEntry
- docIn.seek(skipper.getDocPointer()); // now point to the block we want to search
- // even if freqBuffer were not read from the previous block, we will mark them as read,
- // as we don't need to skip the previous block freqBuffer in refillDocs,
- // as we have already positioned docIn where in needs to be.
- isFreqsRead = true;
- }
- // next time we call advance, this is used to
- // foresee whether skipper is necessary.
- nextSkipDoc = skipper.getNextSkipDoc();
- }
- if (docBufferUpto == BLOCK_SIZE) {
- refillDocs();
- }
-
- // Now scan... this is an inlined/pared down version
- // of nextDoc():
- long doc;
- while (true) {
- doc = docBuffer[docBufferUpto];
-
- if (doc >= target) {
- break;
- }
- ++docBufferUpto;
- }
-
- docBufferUpto++;
- return this.doc = (int) doc;
- }
-
- @Override
- public long cost() {
- return docFreq;
- }
- }
-
- // Also handles payloads + offsets
- final class EverythingEnum extends PostingsEnum {
-
- final ForUtil forUtil = new ForUtil();
- final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);
- final PForUtil pforUtil = new PForUtil(forUtil);
-
- private final long[] docBuffer = new long[BLOCK_SIZE + 1];
- private final long[] freqBuffer = new long[BLOCK_SIZE + 1];
- private final long[] posDeltaBuffer = new long[BLOCK_SIZE];
-
- private final long[] payloadLengthBuffer;
- private final long[] offsetStartDeltaBuffer;
- private final long[] offsetLengthBuffer;
-
- private byte[] payloadBytes;
- private int payloadByteUpto;
- private int payloadLength;
-
- private int lastStartOffset;
- private int startOffset;
- private int endOffset;
-
- private int docBufferUpto;
- private int posBufferUpto;
-
- private Lucene84SkipReader skipper;
- private boolean skipped;
-
- final IndexInput startDocIn;
-
- IndexInput docIn;
- final IndexInput posIn;
- final IndexInput payIn;
- final BytesRef payload;
-
- final boolean indexHasOffsets;
- final boolean indexHasPayloads;
-
- private int docFreq; // number of docs in this posting list
- private long totalTermFreq; // number of positions in this posting list
- private int blockUpto; // number of docs in or before the current block
- private int doc; // doc we last read
- private long accum; // accumulator for doc deltas
- private int freq; // freq we last read
- private int position; // current position
-
- // how many positions "behind" we are; nextPosition must
- // skip these to "catch up":
- private int posPendingCount;
-
- // Lazy pos seek: if != -1 then we must seek to this FP
- // before reading positions:
- private long posPendingFP;
-
- // Lazy pay seek: if != -1 then we must seek to this FP
- // before reading payloads/offsets:
- private long payPendingFP;
-
- // Where this term's postings start in the .doc file:
- private long docTermStartFP;
-
- // Where this term's postings start in the .pos file:
- private long posTermStartFP;
-
- // Where this term's payloads/offsets start in the .pay
- // file:
- private long payTermStartFP;
-
- // File pointer where the last (vInt encoded) pos delta
- // block is. We need this to know whether to bulk
- // decode vs vInt decode the block:
- private long lastPosBlockFP;
-
- // Where this term's skip data starts (after
- // docTermStartFP) in the .doc file (or -1 if there is
- // no skip data for this term):
- private long skipOffset;
-
- private int nextSkipDoc;
-
- private boolean needsOffsets; // true if we actually need offsets
- private boolean needsPayloads; // true if we actually need payloads
- private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1
-
- public EverythingEnum(FieldInfo fieldInfo) throws IOException {
- indexHasOffsets =
- fieldInfo
- .getIndexOptions()
- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
- >= 0;
- indexHasPayloads = fieldInfo.hasPayloads();
-
- this.startDocIn = Lucene84PostingsReader.this.docIn;
- this.docIn = null;
- this.posIn = Lucene84PostingsReader.this.posIn.clone();
- if (indexHasOffsets || indexHasPayloads) {
- this.payIn = Lucene84PostingsReader.this.payIn.clone();
- } else {
- this.payIn = null;
- }
- if (indexHasOffsets) {
- offsetStartDeltaBuffer = new long[BLOCK_SIZE];
- offsetLengthBuffer = new long[BLOCK_SIZE];
- } else {
- offsetStartDeltaBuffer = null;
- offsetLengthBuffer = null;
- startOffset = -1;
- endOffset = -1;
- }
-
- if (indexHasPayloads) {
- payloadLengthBuffer = new long[BLOCK_SIZE];
- payloadBytes = new byte[128];
- payload = new BytesRef();
- } else {
- payloadLengthBuffer = null;
- payloadBytes = null;
- payload = null;
- }
-
- // We set the last element of docBuffer to NO_MORE_DOCS, it helps save conditionals in
- // advance()
- docBuffer[BLOCK_SIZE] = NO_MORE_DOCS;
- }
-
- public boolean canReuse(IndexInput docIn, FieldInfo fieldInfo) {
- return docIn == startDocIn
- && indexHasOffsets
- == (fieldInfo
- .getIndexOptions()
- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
- >= 0)
- && indexHasPayloads == fieldInfo.hasPayloads();
- }
-
- public EverythingEnum reset(IntBlockTermState termState, int flags) throws IOException {
- docFreq = termState.docFreq;
- docTermStartFP = termState.docStartFP;
- posTermStartFP = termState.posStartFP;
- payTermStartFP = termState.payStartFP;
- skipOffset = termState.skipOffset;
- totalTermFreq = termState.totalTermFreq;
- singletonDocID = termState.singletonDocID;
- if (docFreq > 1) {
- if (docIn == null) {
- // lazy init
- docIn = startDocIn.clone();
- }
- docIn.seek(docTermStartFP);
- }
- posPendingFP = posTermStartFP;
- payPendingFP = payTermStartFP;
- posPendingCount = 0;
- if (termState.totalTermFreq < BLOCK_SIZE) {
- lastPosBlockFP = posTermStartFP;
- } else if (termState.totalTermFreq == BLOCK_SIZE) {
- lastPosBlockFP = -1;
- } else {
- lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset;
- }
-
- this.needsOffsets = PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS);
- this.needsPayloads = PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS);
-
- doc = -1;
- accum = 0;
- blockUpto = 0;
- if (docFreq > BLOCK_SIZE) {
- nextSkipDoc = BLOCK_SIZE - 1; // we won't skip if target is found in first block
- } else {
- nextSkipDoc = NO_MORE_DOCS; // not enough docs for skipping
- }
- docBufferUpto = BLOCK_SIZE;
- skipped = false;
- return this;
- }
-
- @Override
- public int freq() throws IOException {
- return freq;
- }
-
- @Override
- public int docID() {
- return doc;
- }
-
- private void refillDocs() throws IOException {
- final int left = docFreq - blockUpto;
- assert left >= 0;
-
- if (left >= BLOCK_SIZE) {
- forDeltaUtil.decodeAndPrefixSum(docIn, accum, docBuffer);
- pforUtil.decode(docIn, freqBuffer);
- blockUpto += BLOCK_SIZE;
- } else if (docFreq == 1) {
- docBuffer[0] = singletonDocID;
- freqBuffer[0] = totalTermFreq;
- docBuffer[1] = NO_MORE_DOCS;
- blockUpto++;
- } else {
- readVIntBlock(docIn, docBuffer, freqBuffer, left, true);
- prefixSum(docBuffer, left, accum);
- docBuffer[left] = NO_MORE_DOCS;
- blockUpto += left;
- }
- accum = docBuffer[BLOCK_SIZE - 1];
- docBufferUpto = 0;
- assert docBuffer[BLOCK_SIZE] == NO_MORE_DOCS;
- }
-
- private void refillPositions() throws IOException {
- if (posIn.getFilePointer() == lastPosBlockFP) {
- final int count = (int) (totalTermFreq % BLOCK_SIZE);
- int payloadLength = 0;
- int offsetLength = 0;
- payloadByteUpto = 0;
- for (int i = 0; i < count; i++) {
- int code = posIn.readVInt();
- if (indexHasPayloads) {
- if ((code & 1) != 0) {
- payloadLength = posIn.readVInt();
- }
- payloadLengthBuffer[i] = payloadLength;
- posDeltaBuffer[i] = code >>> 1;
- if (payloadLength != 0) {
- if (payloadByteUpto + payloadLength > payloadBytes.length) {
- payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payloadLength);
- }
- posIn.readBytes(payloadBytes, payloadByteUpto, payloadLength);
- payloadByteUpto += payloadLength;
- }
- } else {
- posDeltaBuffer[i] = code;
- }
-
- if (indexHasOffsets) {
- int deltaCode = posIn.readVInt();
- if ((deltaCode & 1) != 0) {
- offsetLength = posIn.readVInt();
- }
- offsetStartDeltaBuffer[i] = deltaCode >>> 1;
- offsetLengthBuffer[i] = offsetLength;
- }
- }
- payloadByteUpto = 0;
- } else {
- pforUtil.decode(posIn, posDeltaBuffer);
-
- if (indexHasPayloads) {
- if (needsPayloads) {
- pforUtil.decode(payIn, payloadLengthBuffer);
- int numBytes = payIn.readVInt();
-
- if (numBytes > payloadBytes.length) {
- payloadBytes = ArrayUtil.grow(payloadBytes, numBytes);
- }
- payIn.readBytes(payloadBytes, 0, numBytes);
- } else {
- // this works, because when writing a vint block we always force the first length to be
- // written
- pforUtil.skip(payIn); // skip over lengths
- int numBytes = payIn.readVInt(); // read length of payloadBytes
- payIn.seek(payIn.getFilePointer() + numBytes); // skip over payloadBytes
- }
- payloadByteUpto = 0;
- }
-
- if (indexHasOffsets) {
- if (needsOffsets) {
- pforUtil.decode(payIn, offsetStartDeltaBuffer);
- pforUtil.decode(payIn, offsetLengthBuffer);
- } else {
- // this works, because when writing a vint block we always force the first length to be
- // written
- pforUtil.skip(payIn); // skip over starts
- pforUtil.skip(payIn); // skip over lengths
- }
- }
- }
- }
-
- @Override
- public int nextDoc() throws IOException {
- if (docBufferUpto == BLOCK_SIZE) {
- refillDocs();
- }
-
- doc = (int) docBuffer[docBufferUpto];
- freq = (int) freqBuffer[docBufferUpto];
- posPendingCount += freq;
- docBufferUpto++;
-
- position = 0;
- lastStartOffset = 0;
- return doc;
- }
-
- @Override
- public int advance(int target) throws IOException {
- if (target > nextSkipDoc) {
- if (skipper == null) {
- // Lazy init: first time this enum has ever been used for skipping
- skipper =
- new Lucene84SkipReader(
- docIn.clone(), MAX_SKIP_LEVELS, true, indexHasOffsets, indexHasPayloads);
- }
-
- if (!skipped) {
- assert skipOffset != -1;
- // This is the first time this enum has skipped
- // since reset() was called; load the skip data:
- skipper.init(
- docTermStartFP + skipOffset, docTermStartFP, posTermStartFP, payTermStartFP, docFreq);
- skipped = true;
- }
-
- final int newDocUpto = skipper.skipTo(target) + 1;
-
- if (newDocUpto > blockUpto - BLOCK_SIZE + docBufferUpto) {
- // Skipper moved
- assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;
- blockUpto = newDocUpto;
-
- // Force to read next block
- docBufferUpto = BLOCK_SIZE;
- accum = skipper.getDoc();
- docIn.seek(skipper.getDocPointer());
- posPendingFP = skipper.getPosPointer();
- payPendingFP = skipper.getPayPointer();
- posPendingCount = skipper.getPosBufferUpto();
- lastStartOffset = 0; // new document
- payloadByteUpto = skipper.getPayloadByteUpto();
- }
- nextSkipDoc = skipper.getNextSkipDoc();
- }
- if (docBufferUpto == BLOCK_SIZE) {
- refillDocs();
- }
-
- // Now scan:
- long doc;
- while (true) {
- doc = docBuffer[docBufferUpto];
- freq = (int) freqBuffer[docBufferUpto];
- posPendingCount += freq;
- docBufferUpto++;
-
- if (doc >= target) {
- break;
- }
- }
-
- position = 0;
- lastStartOffset = 0;
- return this.doc = (int) doc;
- }
-
- // TODO: in theory we could avoid loading frq block
- // when not needed, ie, use skip data to load how far to
- // seek the pos pointer ... instead of having to load frq
- // blocks only to sum up how many positions to skip
- private void skipPositions() throws IOException {
- // Skip positions now:
- int toSkip = posPendingCount - freq;
- // if (DEBUG) {
- // System.out.println(" FPR.skipPositions: toSkip=" + toSkip);
- // }
-
- final int leftInBlock = BLOCK_SIZE - posBufferUpto;
- if (toSkip < leftInBlock) {
- int end = posBufferUpto + toSkip;
- while (posBufferUpto < end) {
- if (indexHasPayloads) {
- payloadByteUpto += payloadLengthBuffer[posBufferUpto];
- }
- posBufferUpto++;
- }
- } else {
- toSkip -= leftInBlock;
- while (toSkip >= BLOCK_SIZE) {
- assert posIn.getFilePointer() != lastPosBlockFP;
- pforUtil.skip(posIn);
-
- if (indexHasPayloads) {
- // Skip payloadLength block:
- pforUtil.skip(payIn);
-
- // Skip payloadBytes block:
- int numBytes = payIn.readVInt();
- payIn.seek(payIn.getFilePointer() + numBytes);
- }
-
- if (indexHasOffsets) {
- pforUtil.skip(payIn);
- pforUtil.skip(payIn);
- }
- toSkip -= BLOCK_SIZE;
- }
- refillPositions();
- payloadByteUpto = 0;
- posBufferUpto = 0;
- while (posBufferUpto < toSkip) {
- if (indexHasPayloads) {
- payloadByteUpto += payloadLengthBuffer[posBufferUpto];
- }
- posBufferUpto++;
- }
- }
-
- position = 0;
- lastStartOffset = 0;
- }
-
- @Override
- public int nextPosition() throws IOException {
- assert posPendingCount > 0;
-
- if (posPendingFP != -1) {
- posIn.seek(posPendingFP);
- posPendingFP = -1;
-
- if (payPendingFP != -1 && payIn != null) {
- payIn.seek(payPendingFP);
- payPendingFP = -1;
- }
-
- // Force buffer refill:
- posBufferUpto = BLOCK_SIZE;
- }
-
- if (posPendingCount > freq) {
- skipPositions();
- posPendingCount = freq;
- }
-
- if (posBufferUpto == BLOCK_SIZE) {
- refillPositions();
- posBufferUpto = 0;
- }
- position += posDeltaBuffer[posBufferUpto];
-
- if (indexHasPayloads) {
- payloadLength = (int) payloadLengthBuffer[posBufferUpto];
- payload.bytes = payloadBytes;
- payload.offset = payloadByteUpto;
- payload.length = payloadLength;
- payloadByteUpto += payloadLength;
- }
-
- if (indexHasOffsets) {
- startOffset = lastStartOffset + (int) offsetStartDeltaBuffer[posBufferUpto];
- endOffset = startOffset + (int) offsetLengthBuffer[posBufferUpto];
- lastStartOffset = startOffset;
- }
-
- posBufferUpto++;
- posPendingCount--;
- return position;
- }
-
- @Override
- public int startOffset() {
- return startOffset;
- }
-
- @Override
- public int endOffset() {
- return endOffset;
- }
-
- @Override
- public BytesRef getPayload() {
- if (payloadLength == 0) {
- return null;
- } else {
- return payload;
- }
- }
-
- @Override
- public long cost() {
- return docFreq;
- }
- }
-
- final class BlockImpactsDocsEnum extends ImpactsEnum {
-
- final ForUtil forUtil = new ForUtil();
- final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);
- final PForUtil pforUtil = new PForUtil(forUtil);
-
- private final long[] docBuffer = new long[BLOCK_SIZE + 1];
- private final long[] freqBuffer = new long[BLOCK_SIZE];
-
- private int docBufferUpto;
-
- private final Lucene84ScoreSkipReader skipper;
-
- final IndexInput docIn;
-
- final boolean indexHasFreqs;
-
- private int docFreq; // number of docs in this posting list
- private int blockUpto; // number of documents in or before the current block
- private int doc; // doc we last read
- private long accum; // accumulator for doc deltas
-
- private int nextSkipDoc = -1;
-
- private long seekTo = -1;
-
- // as we read freqBuffer lazily, isFreqsRead shows if freqBuffer are read for the current block
- // always true when we don't have freqBuffer (indexHasFreq=false) or don't need freqBuffer
- // (needsFreq=false)
- private boolean isFreqsRead;
-
- public BlockImpactsDocsEnum(FieldInfo fieldInfo, IntBlockTermState termState)
- throws IOException {
- indexHasFreqs = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
- final boolean indexHasPositions =
- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- final boolean indexHasOffsets =
- fieldInfo
- .getIndexOptions()
- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
- >= 0;
- final boolean indexHasPayloads = fieldInfo.hasPayloads();
-
- this.docIn = Lucene84PostingsReader.this.docIn.clone();
-
- docFreq = termState.docFreq;
- docIn.seek(termState.docStartFP);
-
- doc = -1;
- accum = 0;
- blockUpto = 0;
- docBufferUpto = BLOCK_SIZE;
-
- skipper =
- new Lucene84ScoreSkipReader(
- docIn.clone(), MAX_SKIP_LEVELS, indexHasPositions, indexHasOffsets, indexHasPayloads);
- skipper.init(
- termState.docStartFP + termState.skipOffset,
- termState.docStartFP,
- termState.posStartFP,
- termState.payStartFP,
- docFreq);
-
- // We set the last element of docBuffer to NO_MORE_DOCS, it helps save conditionals in
- // advance()
- docBuffer[BLOCK_SIZE] = NO_MORE_DOCS;
- this.isFreqsRead = true;
- if (indexHasFreqs == false) {
- Arrays.fill(freqBuffer, 1L);
- }
- }
-
- @Override
- public int freq() throws IOException {
- if (isFreqsRead == false) {
- pforUtil.decode(docIn, freqBuffer); // read freqBuffer for this block
- isFreqsRead = true;
- }
- return (int) freqBuffer[docBufferUpto - 1];
- }
-
- @Override
- public int docID() {
- return doc;
- }
-
- private void refillDocs() throws IOException {
- // Check if we skipped reading the previous block of freqBuffer, and if yes, position docIn
- // after it
- if (isFreqsRead == false) {
- pforUtil.skip(docIn);
- isFreqsRead = true;
- }
-
- final int left = docFreq - blockUpto;
- assert left >= 0;
-
- if (left >= BLOCK_SIZE) {
- forDeltaUtil.decodeAndPrefixSum(docIn, accum, docBuffer);
- if (indexHasFreqs) {
- pforUtil.decode(docIn, freqBuffer);
- }
- blockUpto += BLOCK_SIZE;
- } else {
- readVIntBlock(docIn, docBuffer, freqBuffer, left, indexHasFreqs);
- prefixSum(docBuffer, left, accum);
- docBuffer[left] = NO_MORE_DOCS;
- blockUpto += left;
- }
- accum = docBuffer[BLOCK_SIZE - 1];
- docBufferUpto = 0;
- assert docBuffer[BLOCK_SIZE] == NO_MORE_DOCS;
- }
-
- @Override
- public void advanceShallow(int target) throws IOException {
- if (target > nextSkipDoc) {
- // always plus one to fix the result, since skip position in Lucene84SkipReader
- // is a little different from MultiLevelSkipListReader
- final int newDocUpto = skipper.skipTo(target) + 1;
-
- if (newDocUpto >= blockUpto) {
- // Skipper moved
- assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;
- blockUpto = newDocUpto;
-
- // Force to read next block
- docBufferUpto = BLOCK_SIZE;
- accum = skipper.getDoc();
- seekTo = skipper.getDocPointer(); // delay the seek
- }
- // next time we call advance, this is used to
- // foresee whether skipper is necessary.
- nextSkipDoc = skipper.getNextSkipDoc();
- }
- assert nextSkipDoc >= target;
- }
-
- @Override
- public Impacts getImpacts() throws IOException {
- advanceShallow(doc);
- return skipper.getImpacts();
- }
-
- @Override
- public int nextDoc() throws IOException {
- return advance(doc + 1);
- }
-
- @Override
- public int advance(int target) throws IOException {
- if (target > nextSkipDoc) {
- advanceShallow(target);
- }
- if (docBufferUpto == BLOCK_SIZE) {
- if (seekTo >= 0) {
- docIn.seek(seekTo);
- isFreqsRead = true; // reset isFreqsRead
- seekTo = -1;
- }
- refillDocs();
- }
-
- int next = findFirstGreater(docBuffer, target, docBufferUpto);
- this.doc = (int) docBuffer[next];
- docBufferUpto = next + 1;
- return doc;
- }
-
- @Override
- public int nextPosition() throws IOException {
- return -1;
- }
-
- @Override
- public int startOffset() {
- return -1;
- }
-
- @Override
- public int endOffset() {
- return -1;
- }
-
- @Override
- public BytesRef getPayload() {
- return null;
- }
-
- @Override
- public long cost() {
- return docFreq;
- }
- }
-
- final class BlockImpactsPostingsEnum extends ImpactsEnum {
-
- final ForUtil forUtil = new ForUtil();
- final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);
- final PForUtil pforUtil = new PForUtil(forUtil);
-
- private final long[] docBuffer = new long[BLOCK_SIZE];
- private final long[] freqBuffer = new long[BLOCK_SIZE];
- private final long[] posDeltaBuffer = new long[BLOCK_SIZE];
-
- private int docBufferUpto;
- private int posBufferUpto;
-
- private final Lucene84ScoreSkipReader skipper;
-
- final IndexInput docIn;
- final IndexInput posIn;
-
- final boolean indexHasOffsets;
- final boolean indexHasPayloads;
-
- private int docFreq; // number of docs in this posting list
- private long totalTermFreq; // number of positions in this posting list
- private int docUpto; // how many docs we've read
- private int doc; // doc we last read
- private long accum; // accumulator for doc deltas
- private int freq; // freq we last read
- private int position; // current position
-
- // how many positions "behind" we are; nextPosition must
- // skip these to "catch up":
- private int posPendingCount;
-
- // Lazy pos seek: if != -1 then we must seek to this FP
- // before reading positions:
- private long posPendingFP;
-
- // Where this term's postings start in the .doc file:
- private long docTermStartFP;
-
- // Where this term's postings start in the .pos file:
- private long posTermStartFP;
-
- // Where this term's payloads/offsets start in the .pay
- // file:
- private long payTermStartFP;
-
- // File pointer where the last (vInt encoded) pos delta
- // block is. We need this to know whether to bulk
- // decode vs vInt decode the block:
- private long lastPosBlockFP;
-
- private int nextSkipDoc = -1;
-
- private long seekTo = -1;
-
- public BlockImpactsPostingsEnum(FieldInfo fieldInfo, IntBlockTermState termState)
- throws IOException {
- indexHasOffsets =
- fieldInfo
- .getIndexOptions()
- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
- >= 0;
- indexHasPayloads = fieldInfo.hasPayloads();
-
- this.docIn = Lucene84PostingsReader.this.docIn.clone();
-
- this.posIn = Lucene84PostingsReader.this.posIn.clone();
-
- docFreq = termState.docFreq;
- docTermStartFP = termState.docStartFP;
- posTermStartFP = termState.posStartFP;
- payTermStartFP = termState.payStartFP;
- totalTermFreq = termState.totalTermFreq;
- docIn.seek(docTermStartFP);
- posPendingFP = posTermStartFP;
- posPendingCount = 0;
- if (termState.totalTermFreq < BLOCK_SIZE) {
- lastPosBlockFP = posTermStartFP;
- } else if (termState.totalTermFreq == BLOCK_SIZE) {
- lastPosBlockFP = -1;
- } else {
- lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset;
- }
-
- doc = -1;
- accum = 0;
- docUpto = 0;
- docBufferUpto = BLOCK_SIZE;
-
- skipper =
- new Lucene84ScoreSkipReader(
- docIn.clone(), MAX_SKIP_LEVELS, true, indexHasOffsets, indexHasPayloads);
- skipper.init(
- docTermStartFP + termState.skipOffset,
- docTermStartFP,
- posTermStartFP,
- payTermStartFP,
- docFreq);
- }
-
- @Override
- public int freq() throws IOException {
- return freq;
- }
-
- @Override
- public int docID() {
- return doc;
- }
-
- private void refillDocs() throws IOException {
- final int left = docFreq - docUpto;
- assert left >= 0;
-
- if (left >= BLOCK_SIZE) {
- forDeltaUtil.decodeAndPrefixSum(docIn, accum, docBuffer);
- pforUtil.decode(docIn, freqBuffer);
- } else {
- readVIntBlock(docIn, docBuffer, freqBuffer, left, true);
- prefixSum(docBuffer, left, accum);
- docBuffer[left] = NO_MORE_DOCS;
- }
- accum = docBuffer[BLOCK_SIZE - 1];
- docBufferUpto = 0;
- }
-
- private void refillPositions() throws IOException {
- if (posIn.getFilePointer() == lastPosBlockFP) {
- final int count = (int) (totalTermFreq % BLOCK_SIZE);
- int payloadLength = 0;
- for (int i = 0; i < count; i++) {
- int code = posIn.readVInt();
- if (indexHasPayloads) {
- if ((code & 1) != 0) {
- payloadLength = posIn.readVInt();
- }
- posDeltaBuffer[i] = code >>> 1;
- if (payloadLength != 0) {
- posIn.seek(posIn.getFilePointer() + payloadLength);
- }
- } else {
- posDeltaBuffer[i] = code;
- }
- if (indexHasOffsets) {
- if ((posIn.readVInt() & 1) != 0) {
- // offset length changed
- posIn.readVInt();
- }
- }
- }
- } else {
- pforUtil.decode(posIn, posDeltaBuffer);
- }
- }
-
- @Override
- public void advanceShallow(int target) throws IOException {
- if (target > nextSkipDoc) {
- // always plus one to fix the result, since skip position in Lucene84SkipReader
- // is a little different from MultiLevelSkipListReader
- final int newDocUpto = skipper.skipTo(target) + 1;
-
- if (newDocUpto > docUpto) {
- // Skipper moved
- assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;
- docUpto = newDocUpto;
-
- // Force to read next block
- docBufferUpto = BLOCK_SIZE;
- accum = skipper.getDoc();
- posPendingFP = skipper.getPosPointer();
- posPendingCount = skipper.getPosBufferUpto();
- seekTo = skipper.getDocPointer(); // delay the seek
- }
- // next time we call advance, this is used to
- // foresee whether skipper is necessary.
- nextSkipDoc = skipper.getNextSkipDoc();
- }
- assert nextSkipDoc >= target;
- }
-
- @Override
- public Impacts getImpacts() throws IOException {
- advanceShallow(doc);
- return skipper.getImpacts();
- }
-
- @Override
- public int nextDoc() throws IOException {
- return advance(doc + 1);
- }
-
- @Override
- public int advance(int target) throws IOException {
- if (target > nextSkipDoc) {
- advanceShallow(target);
- }
- if (docBufferUpto == BLOCK_SIZE) {
- if (seekTo >= 0) {
- docIn.seek(seekTo);
- seekTo = -1;
- }
- refillDocs();
- }
-
- int next = findFirstGreater(docBuffer, target, docBufferUpto);
- if (next == BLOCK_SIZE) {
- return doc = NO_MORE_DOCS;
- }
- this.doc = (int) docBuffer[next];
- this.freq = (int) freqBuffer[next];
- for (int i = docBufferUpto; i <= next; ++i) {
- posPendingCount += freqBuffer[i];
- }
- docUpto += next - docBufferUpto + 1;
- docBufferUpto = next + 1;
- position = 0;
- return doc;
- }
-
- // TODO: in theory we could avoid loading frq block
- // when not needed, ie, use skip data to load how far to
- // seek the pos pointer ... instead of having to load frq
- // blocks only to sum up how many positions to skip
- private void skipPositions() throws IOException {
- // Skip positions now:
- int toSkip = posPendingCount - freq;
-
- final int leftInBlock = BLOCK_SIZE - posBufferUpto;
- if (toSkip < leftInBlock) {
- posBufferUpto += toSkip;
- } else {
- toSkip -= leftInBlock;
- while (toSkip >= BLOCK_SIZE) {
- assert posIn.getFilePointer() != lastPosBlockFP;
- pforUtil.skip(posIn);
- toSkip -= BLOCK_SIZE;
- }
- refillPositions();
- posBufferUpto = toSkip;
- }
-
- position = 0;
- }
-
- @Override
- public int nextPosition() throws IOException {
- assert posPendingCount > 0;
-
- if (posPendingFP != -1) {
- posIn.seek(posPendingFP);
- posPendingFP = -1;
-
- // Force buffer refill:
- posBufferUpto = BLOCK_SIZE;
- }
-
- if (posPendingCount > freq) {
- skipPositions();
- posPendingCount = freq;
- }
-
- if (posBufferUpto == BLOCK_SIZE) {
- refillPositions();
- posBufferUpto = 0;
- }
- position += posDeltaBuffer[posBufferUpto++];
-
- posPendingCount--;
- return position;
- }
-
- @Override
- public int startOffset() {
- return -1;
- }
-
- @Override
- public int endOffset() {
- return -1;
- }
-
- @Override
- public BytesRef getPayload() {
- return null;
- }
-
- @Override
- public long cost() {
- return docFreq;
- }
- }
-
- final class BlockImpactsEverythingEnum extends ImpactsEnum {
-
- final ForUtil forUtil = new ForUtil();
- final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);
- final PForUtil pforUtil = new PForUtil(forUtil);
-
- private final long[] docBuffer = new long[BLOCK_SIZE];
- private final long[] freqBuffer = new long[BLOCK_SIZE];
- private final long[] posDeltaBuffer = new long[BLOCK_SIZE];
-
- private final long[] payloadLengthBuffer;
- private final long[] offsetStartDeltaBuffer;
- private final long[] offsetLengthBuffer;
-
- private byte[] payloadBytes;
- private int payloadByteUpto;
- private int payloadLength;
-
- private int lastStartOffset;
- private int startOffset = -1;
- private int endOffset = -1;
-
- private int docBufferUpto;
- private int posBufferUpto;
-
- private final Lucene84ScoreSkipReader skipper;
-
- final IndexInput docIn;
- final IndexInput posIn;
- final IndexInput payIn;
- final BytesRef payload;
-
- final boolean indexHasFreq;
- final boolean indexHasPos;
- final boolean indexHasOffsets;
- final boolean indexHasPayloads;
-
- private int docFreq; // number of docs in this posting list
- private long totalTermFreq; // number of positions in this posting list
- private int docUpto; // how many docs we've read
- private int posDocUpTo; // for how many docs we've read positions, offsets, and payloads
- private int doc; // doc we last read
- private long accum; // accumulator for doc deltas
- private int position; // current position
-
- // how many positions "behind" we are; nextPosition must
- // skip these to "catch up":
- private int posPendingCount;
-
- // Lazy pos seek: if != -1 then we must seek to this FP
- // before reading positions:
- private long posPendingFP;
-
- // Lazy pay seek: if != -1 then we must seek to this FP
- // before reading payloads/offsets:
- private long payPendingFP;
-
- // Where this term's postings start in the .doc file:
- private long docTermStartFP;
-
- // Where this term's postings start in the .pos file:
- private long posTermStartFP;
-
- // Where this term's payloads/offsets start in the .pay
- // file:
- private long payTermStartFP;
-
- // File pointer where the last (vInt encoded) pos delta
- // block is. We need this to know whether to bulk
- // decode vs vInt decode the block:
- private long lastPosBlockFP;
-
- private int nextSkipDoc = -1;
-
- private final boolean needsPositions;
- private final boolean needsOffsets; // true if we actually need offsets
- private final boolean needsPayloads; // true if we actually need payloads
-
- private boolean
- isFreqsRead; // shows if freqBuffer for the current doc block are read into freqBuffer
-
- private long seekTo = -1;
-
- public BlockImpactsEverythingEnum(FieldInfo fieldInfo, IntBlockTermState termState, int flags)
- throws IOException {
- indexHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
- indexHasPos =
- fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- indexHasOffsets =
- fieldInfo
- .getIndexOptions()
- .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
- >= 0;
- indexHasPayloads = fieldInfo.hasPayloads();
-
- needsPositions = PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS);
- needsOffsets = PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS);
- needsPayloads = PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS);
-
- this.docIn = Lucene84PostingsReader.this.docIn.clone();
-
- if (indexHasPos && needsPositions) {
- this.posIn = Lucene84PostingsReader.this.posIn.clone();
- } else {
- this.posIn = null;
- }
-
- if ((indexHasOffsets && needsOffsets) || (indexHasPayloads && needsPayloads)) {
- this.payIn = Lucene84PostingsReader.this.payIn.clone();
- } else {
- this.payIn = null;
- }
-
- if (indexHasOffsets) {
- offsetStartDeltaBuffer = new long[BLOCK_SIZE];
- offsetLengthBuffer = new long[BLOCK_SIZE];
- } else {
- offsetStartDeltaBuffer = null;
- offsetLengthBuffer = null;
- startOffset = -1;
- endOffset = -1;
- }
-
- if (indexHasPayloads) {
- payloadLengthBuffer = new long[BLOCK_SIZE];
- payloadBytes = new byte[128];
- payload = new BytesRef();
- } else {
- payloadLengthBuffer = null;
- payloadBytes = null;
- payload = null;
- }
-
- docFreq = termState.docFreq;
- docTermStartFP = termState.docStartFP;
- posTermStartFP = termState.posStartFP;
- payTermStartFP = termState.payStartFP;
- totalTermFreq = termState.totalTermFreq;
- docIn.seek(docTermStartFP);
- posPendingFP = posTermStartFP;
- payPendingFP = payTermStartFP;
- posPendingCount = 0;
- if (termState.totalTermFreq < BLOCK_SIZE) {
- lastPosBlockFP = posTermStartFP;
- } else if (termState.totalTermFreq == BLOCK_SIZE) {
- lastPosBlockFP = -1;
- } else {
- lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset;
- }
-
- doc = -1;
- accum = 0;
- docUpto = 0;
- posDocUpTo = 0;
- isFreqsRead = true;
- docBufferUpto = BLOCK_SIZE;
-
- skipper =
- new Lucene84ScoreSkipReader(
- docIn.clone(), MAX_SKIP_LEVELS, indexHasPos, indexHasOffsets, indexHasPayloads);
- skipper.init(
- docTermStartFP + termState.skipOffset,
- docTermStartFP,
- posTermStartFP,
- payTermStartFP,
- docFreq);
-
- if (indexHasFreq == false) {
- for (int i = 0; i < BLOCK_SIZE; ++i) {
- freqBuffer[i] = 1;
- }
- }
- }
-
- @Override
- public int freq() throws IOException {
- if (indexHasFreq && (isFreqsRead == false)) {
- pforUtil.decode(docIn, freqBuffer); // read freqBuffer for this block
- isFreqsRead = true;
- }
- return (int) freqBuffer[docBufferUpto - 1];
- }
-
- @Override
- public int docID() {
- return doc;
- }
-
- private void refillDocs() throws IOException {
- if (indexHasFreq) {
- if (isFreqsRead == false) { // previous freq block was not read
- // check if we need to load the previous freq block to catch up on positions or we can
- // skip it
- if (indexHasPos && needsPositions && (posDocUpTo < docUpto)) {
- pforUtil.decode(docIn, freqBuffer); // load the previous freq block
- } else {
- pforUtil.skip(docIn); // skip it
- }
- isFreqsRead = true;
- }
- if (indexHasPos && needsPositions) {
- while (posDocUpTo
- < docUpto) { // catch on positions, bring posPendingCount upto the current doc
- posPendingCount += freqBuffer[docBufferUpto - (docUpto - posDocUpTo)];
- posDocUpTo++;
- }
- }
- }
-
- final int left = docFreq - docUpto;
- assert left >= 0;
-
- if (left >= BLOCK_SIZE) {
- forDeltaUtil.decodeAndPrefixSum(docIn, accum, docBuffer);
- if (indexHasFreq) {
- isFreqsRead =
- false; // freq block will be loaded lazily when necessary, we don't load it here
- }
- } else {
- readVIntBlock(docIn, docBuffer, freqBuffer, left, indexHasFreq);
- prefixSum(docBuffer, left, accum);
- docBuffer[left] = NO_MORE_DOCS;
- }
- accum = docBuffer[BLOCK_SIZE - 1];
- docBufferUpto = 0;
- }
-
- private void refillPositions() throws IOException {
- if (posIn.getFilePointer() == lastPosBlockFP) {
- final int count = (int) (totalTermFreq % BLOCK_SIZE);
- int payloadLength = 0;
- int offsetLength = 0;
- payloadByteUpto = 0;
- for (int i = 0; i < count; i++) {
- int code = posIn.readVInt();
- if (indexHasPayloads) {
- if ((code & 1) != 0) {
- payloadLength = posIn.readVInt();
- }
- payloadLengthBuffer[i] = payloadLength;
- posDeltaBuffer[i] = code >>> 1;
- if (payloadLength != 0) {
- if (payloadByteUpto + payloadLength > payloadBytes.length) {
- payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payloadLength);
- }
- posIn.readBytes(payloadBytes, payloadByteUpto, payloadLength);
- payloadByteUpto += payloadLength;
- }
- } else {
- posDeltaBuffer[i] = code;
- }
-
- if (indexHasOffsets) {
- int deltaCode = posIn.readVInt();
- if ((deltaCode & 1) != 0) {
- offsetLength = posIn.readVInt();
- }
- offsetStartDeltaBuffer[i] = deltaCode >>> 1;
- offsetLengthBuffer[i] = offsetLength;
- }
- }
- payloadByteUpto = 0;
- } else {
- pforUtil.decode(posIn, posDeltaBuffer);
-
- if (indexHasPayloads && payIn != null) {
- if (needsPayloads) {
- pforUtil.decode(payIn, payloadLengthBuffer);
- int numBytes = payIn.readVInt();
-
- if (numBytes > payloadBytes.length) {
- payloadBytes = ArrayUtil.grow(payloadBytes, numBytes);
- }
- payIn.readBytes(payloadBytes, 0, numBytes);
- } else {
- // this works, because when writing a vint block we always force the first length to be
- // written
- pforUtil.skip(payIn); // skip over lengths
- int numBytes = payIn.readVInt(); // read length of payloadBytes
- payIn.seek(payIn.getFilePointer() + numBytes); // skip over payloadBytes
- }
- payloadByteUpto = 0;
- }
-
- if (indexHasOffsets && payIn != null) {
- if (needsOffsets) {
- pforUtil.decode(payIn, offsetStartDeltaBuffer);
- pforUtil.decode(payIn, offsetLengthBuffer);
- } else {
- // this works, because when writing a vint block we always force the first length to be
- // written
- pforUtil.skip(payIn); // skip over starts
- pforUtil.skip(payIn); // skip over lengths
- }
- }
- }
- }
-
- @Override
- public void advanceShallow(int target) throws IOException {
- if (target > nextSkipDoc) {
- // always plus one to fix the result, since skip position in Lucene84SkipReader
- // is a little different from MultiLevelSkipListReader
- final int newDocUpto = skipper.skipTo(target) + 1;
-
- if (newDocUpto > docUpto) {
- // Skipper moved
- assert newDocUpto % BLOCK_SIZE == 0 : "got " + newDocUpto;
- docUpto = newDocUpto;
- posDocUpTo = docUpto;
-
- // Force to read next block
- docBufferUpto = BLOCK_SIZE;
- accum = skipper.getDoc();
- posPendingFP = skipper.getPosPointer();
- payPendingFP = skipper.getPayPointer();
- posPendingCount = skipper.getPosBufferUpto();
- lastStartOffset = 0; // new document
- payloadByteUpto = skipper.getPayloadByteUpto(); // actually, this is just lastSkipEntry
- seekTo = skipper.getDocPointer(); // delay the seek
- }
- // next time we call advance, this is used to
- // foresee whether skipper is necessary.
- nextSkipDoc = skipper.getNextSkipDoc();
- }
- assert nextSkipDoc >= target;
- }
-
- @Override
- public Impacts getImpacts() throws IOException {
- advanceShallow(doc);
- return skipper.getImpacts();
- }
-
- @Override
- public int nextDoc() throws IOException {
- return advance(doc + 1);
- }
-
- @Override
- public int advance(int target) throws IOException {
- if (target > nextSkipDoc) {
- advanceShallow(target);
- }
- if (docBufferUpto == BLOCK_SIZE) {
- if (seekTo >= 0) {
- docIn.seek(seekTo);
- seekTo = -1;
- isFreqsRead = true; // reset isFreqsRead
- }
- refillDocs();
- }
-
- // Now scan:
- long doc;
- while (true) {
- doc = docBuffer[docBufferUpto];
- docBufferUpto++;
- docUpto++;
-
- if (doc >= target) {
- break;
- }
-
- if (docBufferUpto == BLOCK_SIZE) {
- return this.doc = NO_MORE_DOCS;
- }
- }
- position = 0;
- lastStartOffset = 0;
-
- return this.doc = (int) doc;
- }
-
- // TODO: in theory we could avoid loading frq block
- // when not needed, ie, use skip data to load how far to
- // seek the pos pointer ... instead of having to load frq
- // blocks only to sum up how many positions to skip
- private void skipPositions() throws IOException {
- // Skip positions now:
- int toSkip = posPendingCount - (int) freqBuffer[docBufferUpto - 1];
- // if (DEBUG) {
- // System.out.println(" FPR.skipPositions: toSkip=" + toSkip);
- // }
-
- final int leftInBlock = BLOCK_SIZE - posBufferUpto;
- if (toSkip < leftInBlock) {
- int end = posBufferUpto + toSkip;
- while (posBufferUpto < end) {
- if (indexHasPayloads) {
- payloadByteUpto += payloadLengthBuffer[posBufferUpto];
- }
- posBufferUpto++;
- }
- } else {
- toSkip -= leftInBlock;
- while (toSkip >= BLOCK_SIZE) {
- assert posIn.getFilePointer() != lastPosBlockFP;
- pforUtil.skip(posIn);
-
- if (indexHasPayloads && payIn != null) {
- // Skip payloadLength block:
- pforUtil.skip(payIn);
-
- // Skip payloadBytes block:
- int numBytes = payIn.readVInt();
- payIn.seek(payIn.getFilePointer() + numBytes);
- }
-
- if (indexHasOffsets && payIn != null) {
- pforUtil.skip(payIn);
- pforUtil.skip(payIn);
- }
- toSkip -= BLOCK_SIZE;
- }
- refillPositions();
- payloadByteUpto = 0;
- posBufferUpto = 0;
- while (posBufferUpto < toSkip) {
- if (indexHasPayloads) {
- payloadByteUpto += payloadLengthBuffer[posBufferUpto];
- }
- posBufferUpto++;
- }
- }
-
- position = 0;
- lastStartOffset = 0;
- }
-
- @Override
- public int nextPosition() throws IOException {
- if (indexHasPos == false || needsPositions == false) {
- return -1;
- }
-
- if (isFreqsRead == false) {
- pforUtil.decode(docIn, freqBuffer); // read freqBuffer for this docs block
- isFreqsRead = true;
- }
- while (posDocUpTo < docUpto) { // bring posPendingCount upto the current doc
- posPendingCount += freqBuffer[docBufferUpto - (docUpto - posDocUpTo)];
- posDocUpTo++;
- }
-
- assert posPendingCount > 0;
-
- if (posPendingFP != -1) {
- posIn.seek(posPendingFP);
- posPendingFP = -1;
-
- if (payPendingFP != -1 && payIn != null) {
- payIn.seek(payPendingFP);
- payPendingFP = -1;
- }
-
- // Force buffer refill:
- posBufferUpto = BLOCK_SIZE;
- }
-
- if (posPendingCount > freqBuffer[docBufferUpto - 1]) {
- skipPositions();
- posPendingCount = (int) freqBuffer[docBufferUpto - 1];
- }
-
- if (posBufferUpto == BLOCK_SIZE) {
- refillPositions();
- posBufferUpto = 0;
- }
- position += posDeltaBuffer[posBufferUpto];
-
- if (indexHasPayloads) {
- payloadLength = (int) payloadLengthBuffer[posBufferUpto];
- payload.bytes = payloadBytes;
- payload.offset = payloadByteUpto;
- payload.length = payloadLength;
- payloadByteUpto += payloadLength;
- }
-
- if (indexHasOffsets && needsOffsets) {
- startOffset = lastStartOffset + (int) offsetStartDeltaBuffer[posBufferUpto];
- endOffset = startOffset + (int) offsetLengthBuffer[posBufferUpto];
- lastStartOffset = startOffset;
- }
-
- posBufferUpto++;
- posPendingCount--;
- return position;
- }
-
- @Override
- public int startOffset() {
- return startOffset;
- }
-
- @Override
- public int endOffset() {
- return endOffset;
- }
-
- @Override
- public BytesRef getPayload() {
- if (payloadLength == 0) {
- return null;
- } else {
- return payload;
- }
- }
-
- @Override
- public long cost() {
- return docFreq;
- }
- }
-
- @Override
- public void checkIntegrity() throws IOException {
- if (docIn != null) {
- CodecUtil.checksumEntireFile(docIn);
- }
- if (posIn != null) {
- CodecUtil.checksumEntireFile(posIn);
- }
- if (payIn != null) {
- CodecUtil.checksumEntireFile(payIn);
- }
- }
-
- @Override
- public String toString() {
- return getClass().getSimpleName()
- + "(positions="
- + (posIn != null)
- + ",payloads="
- + (payIn != null)
- + ")";
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84ScoreSkipReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84ScoreSkipReader.java
deleted file mode 100644
index 6a186afdc3e..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84ScoreSkipReader.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene84;
-
-import java.io.IOException;
-import java.util.Arrays;
-import org.apache.lucene.index.FreqAndNormBuffer;
-import org.apache.lucene.index.Impacts;
-import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.ArrayUtil;
-
-final class Lucene84ScoreSkipReader extends Lucene84SkipReader {
-
- private final byte[][] impactData;
- private final int[] impactDataLength;
- private final ByteArrayDataInput badi = new ByteArrayDataInput();
- private final Impacts impacts;
- private int numLevels = 1;
- private final FreqAndNormBuffer[] perLevelImpacts;
-
- public Lucene84ScoreSkipReader(
- IndexInput skipStream,
- int maxSkipLevels,
- boolean hasPos,
- boolean hasOffsets,
- boolean hasPayloads) {
- super(skipStream, maxSkipLevels, hasPos, hasOffsets, hasPayloads);
- this.impactData = new byte[maxSkipLevels][];
- Arrays.fill(impactData, new byte[0]);
- this.impactDataLength = new int[maxSkipLevels];
- this.perLevelImpacts = new FreqAndNormBuffer[maxSkipLevels];
- for (int i = 0; i < perLevelImpacts.length; ++i) {
- perLevelImpacts[i] = new FreqAndNormBuffer();
- perLevelImpacts[i].add(Integer.MAX_VALUE, 1L);
- }
- impacts =
- new Impacts() {
-
- @Override
- public int numLevels() {
- return numLevels;
- }
-
- @Override
- public int getDocIdUpTo(int level) {
- return skipDoc[level];
- }
-
- @Override
- public FreqAndNormBuffer getImpacts(int level) {
- assert level < numLevels;
- if (impactDataLength[level] > 0) {
- badi.reset(impactData[level], 0, impactDataLength[level]);
- perLevelImpacts[level] = readImpacts(badi, perLevelImpacts[level]);
- impactDataLength[level] = 0;
- }
- return perLevelImpacts[level];
- }
- };
- }
-
- @Override
- public int skipTo(int target) throws IOException {
- int result = super.skipTo(target);
- if (numberOfSkipLevels > 0) {
- numLevels = numberOfSkipLevels;
- } else {
- // End of postings don't have skip data anymore, so we fill with dummy data
- // like SlowImpactsEnum.
- numLevels = 1;
- perLevelImpacts[0].size = 1;
- perLevelImpacts[0].freqs[0] = Integer.MAX_VALUE;
- perLevelImpacts[0].norms[0] = 1L;
- impactDataLength[0] = 0;
- }
- return result;
- }
-
- Impacts getImpacts() {
- return impacts;
- }
-
- @Override
- protected void readImpacts(int level, IndexInput skipStream) throws IOException {
- int length = skipStream.readVInt();
- if (impactData[level].length < length) {
- impactData[level] = new byte[ArrayUtil.oversize(length, Byte.BYTES)];
- }
- skipStream.readBytes(impactData[level], 0, length);
- impactDataLength[level] = length;
- }
-
- static FreqAndNormBuffer readImpacts(ByteArrayDataInput in, FreqAndNormBuffer reuse) {
- int maxNumImpacts = in.length(); // at most one impact per byte
- reuse.growNoCopy(maxNumImpacts);
-
- int freq = 0;
- long norm = 0;
- int size = 0;
- while (in.getPosition() < in.length()) {
- int freqDelta = in.readVInt();
- if ((freqDelta & 0x01) != 0) {
- freq += 1 + (freqDelta >>> 1);
- try {
- norm += 1 + in.readZLong();
- } catch (IOException e) {
- throw new RuntimeException(e); // cannot happen on a BADI
- }
- } else {
- freq += 1 + (freqDelta >>> 1);
- norm++;
- }
- reuse.freqs[size] = freq;
- reuse.norms[size] = norm;
- size++;
- }
- reuse.size = size;
- return reuse;
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84SkipReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84SkipReader.java
deleted file mode 100644
index 78b49e28195..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84SkipReader.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene84;
-
-import java.io.IOException;
-import java.util.Arrays;
-import org.apache.lucene.codecs.MultiLevelSkipListReader;
-import org.apache.lucene.store.IndexInput;
-
-/**
- * Implements the skip list reader for block postings format that stores positions and payloads.
- *
- * <p>Although this skipper uses MultiLevelSkipListReader as an interface, its definition of skip
- * position will be a little different.
- *
- * <p>For example, when skipInterval = blockSize = 3, df = 2*skipInterval = 6,
- *
- * <pre>
- * 0 1 2 3 4 5
- * d d d d d d (posting list)
- * ^ ^ (skip point in MultiLeveSkipWriter)
- * ^ (skip point in Lucene84SkipWriter)
- * </pre>
- *
- * <p>In this case, MultiLevelSkipListReader will use the last document as a skip point, while
- * Lucene84SkipReader should assume no skip point will comes.
- *
- * <p>If we use the interface directly in Lucene84SkipReader, it may silly try to read another skip
- * data after the only skip point is loaded.
- *
- * <p>To illustrate this, we can call skipTo(d[5]), since skip point d[3] has smaller docId, and
- * numSkipped+blockSize== df, the MultiLevelSkipListReader will assume the skip list isn't exhausted
- * yet, and try to load a non-existed skip point
- *
- * <p>Therefore, we'll trim df before passing it to the interface. see trim(int)
- */
-class Lucene84SkipReader extends MultiLevelSkipListReader {
- private long[] docPointer;
- private long[] posPointer;
- private long[] payPointer;
- private int[] posBufferUpto;
- private int[] payloadByteUpto;
-
- private long lastPosPointer;
- private long lastPayPointer;
- private int lastPayloadByteUpto;
- private long lastDocPointer;
- private int lastPosBufferUpto;
-
- public Lucene84SkipReader(
- IndexInput skipStream,
- int maxSkipLevels,
- boolean hasPos,
- boolean hasOffsets,
- boolean hasPayloads) {
- super(skipStream, maxSkipLevels, ForUtil.BLOCK_SIZE, 8);
- docPointer = new long[maxSkipLevels];
- if (hasPos) {
- posPointer = new long[maxSkipLevels];
- posBufferUpto = new int[maxSkipLevels];
- if (hasPayloads) {
- payloadByteUpto = new int[maxSkipLevels];
- } else {
- payloadByteUpto = null;
- }
- if (hasOffsets || hasPayloads) {
- payPointer = new long[maxSkipLevels];
- } else {
- payPointer = null;
- }
- } else {
- posPointer = null;
- }
- }
-
- /**
- * Trim original docFreq to tell skipReader read proper number of skip points.
- *
- * <p>Since our definition in Lucene84Skip* is a little different from MultiLevelSkip* This
- * trimmed docFreq will prevent skipReader from: 1. silly reading a non-existed skip point after
- * the last block boundary 2. moving into the vInt block
- */
- protected int trim(int df) {
- return df % ForUtil.BLOCK_SIZE == 0 ? df - 1 : df;
- }
-
- public void init(
- long skipPointer, long docBasePointer, long posBasePointer, long payBasePointer, int df)
- throws IOException {
- super.init(skipPointer, trim(df));
- lastDocPointer = docBasePointer;
- lastPosPointer = posBasePointer;
- lastPayPointer = payBasePointer;
-
- Arrays.fill(docPointer, docBasePointer);
- if (posPointer != null) {
- Arrays.fill(posPointer, posBasePointer);
- if (payPointer != null) {
- Arrays.fill(payPointer, payBasePointer);
- }
- } else {
- assert posBasePointer == 0;
- }
- }
-
- /**
- * Returns the doc pointer of the doc to which the last call of {@link
- * MultiLevelSkipListReader#skipTo(int)} has skipped.
- */
- public long getDocPointer() {
- return lastDocPointer;
- }
-
- public long getPosPointer() {
- return lastPosPointer;
- }
-
- public int getPosBufferUpto() {
- return lastPosBufferUpto;
- }
-
- public long getPayPointer() {
- return lastPayPointer;
- }
-
- public int getPayloadByteUpto() {
- return lastPayloadByteUpto;
- }
-
- public int getNextSkipDoc() {
- return skipDoc[0];
- }
-
- @Override
- protected void seekChild(int level) throws IOException {
- super.seekChild(level);
- docPointer[level] = lastDocPointer;
- if (posPointer != null) {
- posPointer[level] = lastPosPointer;
- posBufferUpto[level] = lastPosBufferUpto;
- if (payloadByteUpto != null) {
- payloadByteUpto[level] = lastPayloadByteUpto;
- }
- if (payPointer != null) {
- payPointer[level] = lastPayPointer;
- }
- }
- }
-
- @Override
- protected void setLastSkipData(int level) {
- super.setLastSkipData(level);
- lastDocPointer = docPointer[level];
-
- if (posPointer != null) {
- lastPosPointer = posPointer[level];
- lastPosBufferUpto = posBufferUpto[level];
- if (payPointer != null) {
- lastPayPointer = payPointer[level];
- }
- if (payloadByteUpto != null) {
- lastPayloadByteUpto = payloadByteUpto[level];
- }
- }
- }
-
- @Override
- protected int readSkipData(int level, IndexInput skipStream) throws IOException {
- int delta = skipStream.readVInt();
- docPointer[level] += skipStream.readVLong();
-
- if (posPointer != null) {
- posPointer[level] += skipStream.readVLong();
- posBufferUpto[level] = skipStream.readVInt();
-
- if (payloadByteUpto != null) {
- payloadByteUpto[level] = skipStream.readVInt();
- }
-
- if (payPointer != null) {
- payPointer[level] += skipStream.readVLong();
- }
- }
- readImpacts(level, skipStream);
- return delta;
- }
-
- // The default impl skips impacts
- protected void readImpacts(int level, IndexInput skipStream) throws IOException {
- skipStream.skipBytes(skipStream.readVInt());
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/PForUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/PForUtil.java
deleted file mode 100644
index 690bfa501f4..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/PForUtil.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene84;
-
-import java.io.IOException;
-import java.util.Arrays;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.util.packed.PackedInts;
-
-/** Utility class to encode sequences of 128 small positive integers. */
-public final class PForUtil {
-
- static boolean allEqual(long[] l) {
- for (int i = 1; i < ForUtil.BLOCK_SIZE; ++i) {
- if (l[i] != l[0]) {
- return false;
- }
- }
- return true;
- }
-
- private final ForUtil forUtil;
-
- PForUtil(ForUtil forUtil) {
- this.forUtil = forUtil;
- }
-
- /** Encode 128 integers from {@code longs} into {@code out}. */
- void encode(long[] longs, DataOutput out) throws IOException {
- // At most 7 exceptions
- final long[] top8 = new long[8];
- Arrays.fill(top8, -1L);
- for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {
- if (longs[i] > top8[0]) {
- top8[0] = longs[i];
- Arrays.sort(
- top8); // For only 8 entries we just sort on every iteration instead of maintaining a PQ
- }
- }
-
- final int maxBitsRequired = PackedInts.bitsRequired(top8[7]);
- // We store the patch on a byte, so we can't decrease the number of bits required by more than 8
- final int patchedBitsRequired = Math.max(PackedInts.bitsRequired(top8[0]), maxBitsRequired - 8);
- int numExceptions = 0;
- final long maxUnpatchedValue = (1L << patchedBitsRequired) - 1;
- for (int i = 1; i < 8; ++i) {
- if (top8[i] > maxUnpatchedValue) {
- numExceptions++;
- }
- }
- final byte[] exceptions = new byte[numExceptions * 2];
- if (numExceptions > 0) {
- int exceptionCount = 0;
- for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {
- if (longs[i] > maxUnpatchedValue) {
- exceptions[exceptionCount * 2] = (byte) i;
- exceptions[exceptionCount * 2 + 1] = (byte) (longs[i] >>> patchedBitsRequired);
- longs[i] &= maxUnpatchedValue;
- exceptionCount++;
- }
- }
- assert exceptionCount == numExceptions : exceptionCount + " " + numExceptions;
- }
-
- if (allEqual(longs) && maxBitsRequired <= 8) {
- for (int i = 0; i < numExceptions; ++i) {
- exceptions[2 * i + 1] =
- (byte) (Byte.toUnsignedLong(exceptions[2 * i + 1]) << patchedBitsRequired);
- }
- out.writeByte((byte) (numExceptions << 5));
- out.writeVLong(longs[0]);
- } else {
- final int token = (numExceptions << 5) | patchedBitsRequired;
- out.writeByte((byte) token);
- forUtil.encode(longs, patchedBitsRequired, out);
- }
- out.writeBytes(exceptions, exceptions.length);
- }
-
- /** Decode 128 integers into {@code longs}. */
- void decode(DataInput in, long[] longs) throws IOException {
- final int token = Byte.toUnsignedInt(in.readByte());
- final int bitsPerValue = token & 0x1f;
- final int numExceptions = token >>> 5;
- if (bitsPerValue == 0) {
- Arrays.fill(longs, 0, ForUtil.BLOCK_SIZE, in.readVLong());
- } else {
- forUtil.decode(bitsPerValue, in, longs);
- }
- for (int i = 0; i < numExceptions; ++i) {
- longs[Byte.toUnsignedInt(in.readByte())] |=
- Byte.toUnsignedLong(in.readByte()) << bitsPerValue;
- }
- }
-
- /** Skip 128 integers. */
- void skip(DataInput in) throws IOException {
- final int token = Byte.toUnsignedInt(in.readByte());
- final int bitsPerValue = token & 0x1f;
- final int numExceptions = token >>> 5;
- if (bitsPerValue == 0) {
- in.readVLong();
- in.skipBytes((numExceptions << 1));
- } else {
- in.skipBytes(forUtil.numBytes(bitsPerValue) + (numExceptions << 1));
- }
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/gen_ForUtil.py b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/gen_ForUtil.py
deleted file mode 100644
index a4531203ea5..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/gen_ForUtil.py
+++ /dev/null
@@ -1,582 +0,0 @@
-#! /usr/bin/env python
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-"""Code generation for ForUtil.java"""
-
-MAX_SPECIALIZED_BITS_PER_VALUE = 24
-OUTPUT_FILE = "ForUtil.java"
-PRIMITIVE_SIZE = [8, 16, 32]
-HEADER = """// This file has been automatically generated, DO NOT EDIT
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene84;
-
-import java.io.IOException;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.DataOutput;
-
-// Inspired from https://fulmicoton.com/posts/bitpacking/
-// Encodes multiple integers in a long to get SIMD-like speedups.
-// If bitsPerValue <= 8 then we pack 8 ints per long
-// else if bitsPerValue <= 16 we pack 4 ints per long
-// else we pack 2 ints per long
-final class ForUtil {
-
- static final int BLOCK_SIZE = 128;
- private static final int BLOCK_SIZE_LOG2 = 7;
-
- private static long expandMask32(long mask32) {
- return mask32 | (mask32 << 32);
- }
-
- private static long expandMask16(long mask16) {
- return expandMask32(mask16 | (mask16 << 16));
- }
-
- private static long expandMask8(long mask8) {
- return expandMask16(mask8 | (mask8 << 8));
- }
-
- private static long mask32(int bitsPerValue) {
- return expandMask32((1L << bitsPerValue) - 1);
- }
-
- private static long mask16(int bitsPerValue) {
- return expandMask16((1L << bitsPerValue) - 1);
- }
-
- private static long mask8(int bitsPerValue) {
- return expandMask8((1L << bitsPerValue) - 1);
- }
-
- private static void expand8(long[] arr) {
- for (int i = 0; i < 16; ++i) {
- long l = arr[i];
- arr[i] = (l >>> 56) & 0xFFL;
- arr[16 + i] = (l >>> 48) & 0xFFL;
- arr[32 + i] = (l >>> 40) & 0xFFL;
- arr[48 + i] = (l >>> 32) & 0xFFL;
- arr[64 + i] = (l >>> 24) & 0xFFL;
- arr[80 + i] = (l >>> 16) & 0xFFL;
- arr[96 + i] = (l >>> 8) & 0xFFL;
- arr[112 + i] = l & 0xFFL;
- }
- }
-
- private static void expand8To32(long[] arr) {
- for (int i = 0; i < 16; ++i) {
- long l = arr[i];
- arr[i] = (l >>> 24) & 0x000000FF000000FFL;
- arr[16 + i] = (l >>> 16) & 0x000000FF000000FFL;
- arr[32 + i] = (l >>> 8) & 0x000000FF000000FFL;
- arr[48 + i] = l & 0x000000FF000000FFL;
- }
- }
-
- private static void collapse8(long[] arr) {
- for (int i = 0; i < 16; ++i) {
- arr[i] =
- (arr[i] << 56)
- | (arr[16 + i] << 48)
- | (arr[32 + i] << 40)
- | (arr[48 + i] << 32)
- | (arr[64 + i] << 24)
- | (arr[80 + i] << 16)
- | (arr[96 + i] << 8)
- | arr[112 + i];
- }
- }
-
- private static void expand16(long[] arr) {
- for (int i = 0; i < 32; ++i) {
- long l = arr[i];
- arr[i] = (l >>> 48) & 0xFFFFL;
- arr[32 + i] = (l >>> 32) & 0xFFFFL;
- arr[64 + i] = (l >>> 16) & 0xFFFFL;
- arr[96 + i] = l & 0xFFFFL;
- }
- }
-
- private static void expand16To32(long[] arr) {
- for (int i = 0; i < 32; ++i) {
- long l = arr[i];
- arr[i] = (l >>> 16) & 0x0000FFFF0000FFFFL;
- arr[32 + i] = l & 0x0000FFFF0000FFFFL;
- }
- }
-
- private static void collapse16(long[] arr) {
- for (int i = 0; i < 32; ++i) {
- arr[i] = (arr[i] << 48) | (arr[32 + i] << 32) | (arr[64 + i] << 16) | arr[96 + i];
- }
- }
-
- private static void expand32(long[] arr) {
- for (int i = 0; i < 64; ++i) {
- long l = arr[i];
- arr[i] = l >>> 32;
- arr[64 + i] = l & 0xFFFFFFFFL;
- }
- }
-
- private static void collapse32(long[] arr) {
- for (int i = 0; i < 64; ++i) {
- arr[i] = (arr[i] << 32) | arr[64 + i];
- }
- }
-
- private static void prefixSum8(long[] arr, long base) {
- expand8To32(arr);
- prefixSum32(arr, base);
- }
-
- private static void prefixSum16(long[] arr, long base) {
- // We need to move to the next primitive size to avoid overflows
- expand16To32(arr);
- prefixSum32(arr, base);
- }
-
- private static void prefixSum32(long[] arr, long base) {
- arr[0] += base << 32;
- innerPrefixSum32(arr);
- expand32(arr);
- final long l = arr[BLOCK_SIZE / 2 - 1];
- for (int i = BLOCK_SIZE / 2; i < BLOCK_SIZE; ++i) {
- arr[i] += l;
- }
- }
-
- // For some reason unrolling seems to help
- private static void innerPrefixSum32(long[] arr) {
- arr[1] += arr[0];
- arr[2] += arr[1];
- arr[3] += arr[2];
- arr[4] += arr[3];
- arr[5] += arr[4];
- arr[6] += arr[5];
- arr[7] += arr[6];
- arr[8] += arr[7];
- arr[9] += arr[8];
- arr[10] += arr[9];
- arr[11] += arr[10];
- arr[12] += arr[11];
- arr[13] += arr[12];
- arr[14] += arr[13];
- arr[15] += arr[14];
- arr[16] += arr[15];
- arr[17] += arr[16];
- arr[18] += arr[17];
- arr[19] += arr[18];
- arr[20] += arr[19];
- arr[21] += arr[20];
- arr[22] += arr[21];
- arr[23] += arr[22];
- arr[24] += arr[23];
- arr[25] += arr[24];
- arr[26] += arr[25];
- arr[27] += arr[26];
- arr[28] += arr[27];
- arr[29] += arr[28];
- arr[30] += arr[29];
- arr[31] += arr[30];
- arr[32] += arr[31];
- arr[33] += arr[32];
- arr[34] += arr[33];
- arr[35] += arr[34];
- arr[36] += arr[35];
- arr[37] += arr[36];
- arr[38] += arr[37];
- arr[39] += arr[38];
- arr[40] += arr[39];
- arr[41] += arr[40];
- arr[42] += arr[41];
- arr[43] += arr[42];
- arr[44] += arr[43];
- arr[45] += arr[44];
- arr[46] += arr[45];
- arr[47] += arr[46];
- arr[48] += arr[47];
- arr[49] += arr[48];
- arr[50] += arr[49];
- arr[51] += arr[50];
- arr[52] += arr[51];
- arr[53] += arr[52];
- arr[54] += arr[53];
- arr[55] += arr[54];
- arr[56] += arr[55];
- arr[57] += arr[56];
- arr[58] += arr[57];
- arr[59] += arr[58];
- arr[60] += arr[59];
- arr[61] += arr[60];
- arr[62] += arr[61];
- arr[63] += arr[62];
- }
-
- private static void readLELongs(DataInput in, long[] dst, int offset, int length)
- throws IOException {
- in.readLongs(dst, offset, length);
- for (int i = 0; i < length; ++i) {
- dst[offset + i] = Long.reverseBytes(dst[offset + i]);
- }
- }
-
- private final long[] tmp = new long[BLOCK_SIZE / 2];
-
- /** Encode 128 integers from {@code longs} into {@code out}. */
- void encode(long[] longs, int bitsPerValue, DataOutput out) throws IOException {
- final int nextPrimitive;
- final int numLongs;
- if (bitsPerValue <= 8) {
- nextPrimitive = 8;
- numLongs = BLOCK_SIZE / 8;
- collapse8(longs);
- } else if (bitsPerValue <= 16) {
- nextPrimitive = 16;
- numLongs = BLOCK_SIZE / 4;
- collapse16(longs);
- } else {
- nextPrimitive = 32;
- numLongs = BLOCK_SIZE / 2;
- collapse32(longs);
- }
-
- final int numLongsPerShift = bitsPerValue * 2;
- int idx = 0;
- int shift = nextPrimitive - bitsPerValue;
- for (int i = 0; i < numLongsPerShift; ++i) {
- tmp[i] = longs[idx++] << shift;
- }
- for (shift = shift - bitsPerValue; shift >= 0; shift -= bitsPerValue) {
- for (int i = 0; i < numLongsPerShift; ++i) {
- tmp[i] |= longs[idx++] << shift;
- }
- }
-
- final int remainingBitsPerLong = shift + bitsPerValue;
- final long maskRemainingBitsPerLong;
- if (nextPrimitive == 8) {
- maskRemainingBitsPerLong = MASKS8[remainingBitsPerLong];
- } else if (nextPrimitive == 16) {
- maskRemainingBitsPerLong = MASKS16[remainingBitsPerLong];
- } else {
- maskRemainingBitsPerLong = MASKS32[remainingBitsPerLong];
- }
-
- int tmpIdx = 0;
- int remainingBitsPerValue = bitsPerValue;
- while (idx < numLongs) {
- if (remainingBitsPerValue >= remainingBitsPerLong) {
- remainingBitsPerValue -= remainingBitsPerLong;
- tmp[tmpIdx++] |= (longs[idx] >>> remainingBitsPerValue) & maskRemainingBitsPerLong;
- if (remainingBitsPerValue == 0) {
- idx++;
- remainingBitsPerValue = bitsPerValue;
- }
- } else {
- final long mask1, mask2;
- if (nextPrimitive == 8) {
- mask1 = MASKS8[remainingBitsPerValue];
- mask2 = MASKS8[remainingBitsPerLong - remainingBitsPerValue];
- } else if (nextPrimitive == 16) {
- mask1 = MASKS16[remainingBitsPerValue];
- mask2 = MASKS16[remainingBitsPerLong - remainingBitsPerValue];
- } else {
- mask1 = MASKS32[remainingBitsPerValue];
- mask2 = MASKS32[remainingBitsPerLong - remainingBitsPerValue];
- }
- tmp[tmpIdx] |= (longs[idx++] & mask1) << (remainingBitsPerLong - remainingBitsPerValue);
- remainingBitsPerValue = bitsPerValue - remainingBitsPerLong + remainingBitsPerValue;
- tmp[tmpIdx++] |= (longs[idx] >>> remainingBitsPerValue) & mask2;
- }
- }
-
- for (int i = 0; i < numLongsPerShift; ++i) {
- // Java longs are big endian and we want to read little endian longs, so we need to reverse
- // bytes
- long l = Long.reverseBytes(tmp[i]);
- out.writeLong(l);
- }
- }
-
- /** Number of bytes required to encode 128 integers of {@code bitsPerValue} bits per value. */
- int numBytes(int bitsPerValue) throws IOException {
- return bitsPerValue << (BLOCK_SIZE_LOG2 - 3);
- }
-
- private static void decodeSlow(int bitsPerValue, DataInput in, long[] tmp, long[] longs)
- throws IOException {
- final int numLongs = bitsPerValue << 1;
- readLELongs(in, tmp, 0, numLongs);
- final long mask = MASKS32[bitsPerValue];
- int longsIdx = 0;
- int shift = 32 - bitsPerValue;
- for (; shift >= 0; shift -= bitsPerValue) {
- shiftLongs(tmp, numLongs, longs, longsIdx, shift, mask);
- longsIdx += numLongs;
- }
- final int remainingBitsPerLong = shift + bitsPerValue;
- final long mask32RemainingBitsPerLong = MASKS32[remainingBitsPerLong];
- int tmpIdx = 0;
- int remainingBits = remainingBitsPerLong;
- for (; longsIdx < BLOCK_SIZE / 2; ++longsIdx) {
- int b = bitsPerValue - remainingBits;
- long l = (tmp[tmpIdx++] & MASKS32[remainingBits]) << b;
- while (b >= remainingBitsPerLong) {
- b -= remainingBitsPerLong;
- l |= (tmp[tmpIdx++] & mask32RemainingBitsPerLong) << b;
- }
- if (b > 0) {
- l |= (tmp[tmpIdx] >>> (remainingBitsPerLong - b)) & MASKS32[b];
- remainingBits = remainingBitsPerLong - b;
- } else {
- remainingBits = remainingBitsPerLong;
- }
- longs[longsIdx] = l;
- }
- }
-
- /**
- * The pattern that this shiftLongs method applies is recognized by the C2 compiler, which
- * generates SIMD instructions for it in order to shift multiple longs at once.
- */
- private static void shiftLongs(long[] a, int count, long[] b, int bi, int shift, long mask) {
- for (int i = 0; i < count; ++i) {
- b[bi + i] = (a[i] >>> shift) & mask;
- }
- }
-
-"""
-
-
-def writeRemainderWithSIMDOptimize(
- bpv, next_primitive, remaining_bits_per_long, o, num_values, f
-):
- iteration = 1
- num_longs = bpv * num_values / remaining_bits_per_long
- while num_longs % 2 == 0 and num_values % 2 == 0:
- num_longs /= 2
- num_values /= 2
- iteration *= 2
-
- f.write(
- " shiftLongs(tmp, %d, tmp, 0, 0, MASK%d_%d);\n"
- % (iteration * num_longs, next_primitive, remaining_bits_per_long)
- )
- f.write(
- " for (int iter = 0, tmpIdx = 0, longsIdx = %d; iter < %d; ++iter, tmpIdx += %d, longsIdx += %d) {\n"
- % (o, iteration, num_longs, num_values)
- )
- tmp_idx = 0
- b = bpv
- b -= remaining_bits_per_long
- f.write(" long l0 = tmp[tmpIdx + %d] << %d;\n" % (tmp_idx, b))
- tmp_idx += 1
- while b >= remaining_bits_per_long:
- b -= remaining_bits_per_long
- f.write(" l0 |= tmp[tmpIdx + %d] << %d;\n" % (tmp_idx, b))
- tmp_idx += 1
- f.write(" longs[longsIdx + 0] = l0;\n")
- f.write(" }\n")
-
-
-def writeRemainder(bpv, next_primitive, remaining_bits_per_long, o, num_values, f):
- iteration = 1
- num_longs = bpv * num_values / remaining_bits_per_long
- while num_longs % 2 == 0 and num_values % 2 == 0:
- num_longs /= 2
- num_values /= 2
- iteration *= 2
- f.write(
- " for (int iter = 0, tmpIdx = 0, longsIdx = %d; iter < %d; ++iter, tmpIdx += %d, longsIdx += %d) {\n"
- % (o, iteration, num_longs, num_values)
- )
- i = 0
- remaining_bits = 0
- tmp_idx = 0
- for i in range(int(num_values)):
- b = bpv
- if remaining_bits == 0:
- b -= remaining_bits_per_long
- f.write(
- " long l%d = (tmp[tmpIdx + %d] & MASK%d_%d) << %d;\n"
- % (i, tmp_idx, next_primitive, remaining_bits_per_long, b)
- )
- else:
- b -= remaining_bits
- f.write(
- " long l%d = (tmp[tmpIdx + %d] & MASK%d_%d) << %d;\n"
- % (i, tmp_idx, next_primitive, remaining_bits, b)
- )
- tmp_idx += 1
- while b >= remaining_bits_per_long:
- b -= remaining_bits_per_long
- f.write(
- " l%d |= (tmp[tmpIdx + %d] & MASK%d_%d) << %d;\n"
- % (i, tmp_idx, next_primitive, remaining_bits_per_long, b)
- )
- tmp_idx += 1
- if b > 0:
- f.write(
- " l%d |= (tmp[tmpIdx + %d] >>> %d) & MASK%d_%d;\n"
- % (i, tmp_idx, remaining_bits_per_long - b, next_primitive, b)
- )
- remaining_bits = remaining_bits_per_long - b
- f.write(" longs[longsIdx + %d] = l%d;\n" % (i, i))
- f.write(" }\n")
-
-
-def writeDecode(bpv, f):
- next_primitive = 32
- if bpv <= 8:
- next_primitive = 8
- elif bpv <= 16:
- next_primitive = 16
- f.write(
- " private static void decode%d(DataInput in, long[] tmp, long[] longs) throws IOException {\n"
- % bpv
- )
- num_values_per_long = 64 / next_primitive
- if bpv == next_primitive:
- f.write(" readLELongs(in, longs, 0, %d);\n" % (bpv * 2))
- else:
- f.write(" readLELongs(in, tmp, 0, %d);\n" % (bpv * 2))
- shift = next_primitive - bpv
- o = 0
- while shift >= 0:
- f.write(
- " shiftLongs(tmp, %d, longs, %d, %d, MASK%d_%d);\n"
- % (bpv * 2, o, shift, next_primitive, bpv)
- )
- o += bpv * 2
- shift -= bpv
- if shift + bpv > 0:
- if bpv % (next_primitive % bpv) == 0:
- writeRemainderWithSIMDOptimize(
- bpv,
- next_primitive,
- shift + bpv,
- o,
- 128 / num_values_per_long - o,
- f,
- )
- else:
- writeRemainder(
- bpv,
- next_primitive,
- shift + bpv,
- o,
- 128 / num_values_per_long - o,
- f,
- )
- f.write(" }\n")
- f.write("\n")
-
-
-if __name__ == "__main__":
- f = open(OUTPUT_FILE, "w")
- f.write(HEADER)
- for primitive_size in PRIMITIVE_SIZE:
- f.write(
- " private static final long[] MASKS%d = new long[%d];\n"
- % (primitive_size, primitive_size)
- )
- f.write("\n")
- f.write(" static {\n")
- for primitive_size in PRIMITIVE_SIZE:
- f.write(" for (int i = 0; i < %d; ++i) {\n" % primitive_size)
- f.write(" MASKS%d[i] = mask%d(i);\n" % (primitive_size, primitive_size))
- f.write(" }\n")
- f.write(" }\n")
- f.write(
- " // mark values in array as final longs to avoid the cost of reading array, arrays should only be\n"
- )
- f.write(" // used when the idx is a variable\n")
- for primitive_size in PRIMITIVE_SIZE:
- for bpv in range(1, min(MAX_SPECIALIZED_BITS_PER_VALUE + 1, primitive_size)):
- if bpv * 2 != primitive_size or primitive_size == 8:
- f.write(
- " private static final long MASK%d_%d = MASKS%d[%d];\n"
- % (primitive_size, bpv, primitive_size, bpv)
- )
-
- f.write("""
- /** Decode 128 integers into {@code longs}. */
- void decode(int bitsPerValue, DataInput in, long[] longs) throws IOException {
- switch (bitsPerValue) {
-""")
- for bpv in range(1, MAX_SPECIALIZED_BITS_PER_VALUE + 1):
- next_primitive = 32
- if bpv <= 8:
- next_primitive = 8
- elif bpv <= 16:
- next_primitive = 16
- f.write(" case %d:\n" % bpv)
- f.write(" decode%d(in, tmp, longs);\n" % bpv)
- f.write(" expand%d(longs);\n" % next_primitive)
- f.write(" break;\n")
- f.write(" default:\n")
- f.write(" decodeSlow(bitsPerValue, in, tmp, longs);\n")
- f.write(" expand32(longs);\n")
- f.write(" break;\n")
- f.write(" }\n")
- f.write(" }\n")
-
- f.write("""
- /** Delta-decode 128 integers into {@code longs}. */
- void decodeAndPrefixSum(int bitsPerValue, DataInput in, long base, long[] longs)
- throws IOException {
- switch (bitsPerValue) {
-""")
- for bpv in range(1, MAX_SPECIALIZED_BITS_PER_VALUE + 1):
- next_primitive = 32
- if bpv <= 8:
- next_primitive = 8
- elif bpv <= 16:
- next_primitive = 16
- f.write(" case %d:\n" % bpv)
- f.write(" decode%d(in, tmp, longs);\n" % bpv)
- f.write(" prefixSum%d(longs, base);\n" % next_primitive)
- f.write(" break;\n")
- f.write(" default:\n")
- f.write(" decodeSlow(bitsPerValue, in, tmp, longs);\n")
- f.write(" prefixSum32(longs, base);\n")
- f.write(" break;\n")
- f.write(" }\n")
- f.write(" }\n")
-
- f.write("\n")
- for i in range(1, MAX_SPECIALIZED_BITS_PER_VALUE + 1):
- writeDecode(i, f)
-
- f.write("}\n")
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/package-info.java
deleted file mode 100644
index b527657a699..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** Lucene 8.4 file format. */
-package org.apache.lucene.backward_codecs.lucene84;
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86Codec.java
deleted file mode 100644
index 2c56e7676c6..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86Codec.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.lucene.backward_codecs.lucene86;
-
-import java.util.Objects;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat;
-import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;
-import org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat;
-import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.CompoundFormat;
-import org.apache.lucene.codecs.DocValuesFormat;
-import org.apache.lucene.codecs.FieldInfosFormat;
-import org.apache.lucene.codecs.FilterCodec;
-import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.LiveDocsFormat;
-import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.PointsFormat;
-import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.SegmentInfoFormat;
-import org.apache.lucene.codecs.StoredFieldsFormat;
-import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
-import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
-
-/**
- * Implements the Lucene 8.6 index format, with configurable per-field postings and docvalues
- * formats.
- *
- * <p>If you want to reuse functionality of this codec in another codec, extend {@link FilterCodec}.
- *
- * @lucene.experimental
- */
-public class Lucene86Codec extends Codec {
- private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat();
- private final FieldInfosFormat fieldInfosFormat = new Lucene60FieldInfosFormat();
- private final SegmentInfoFormat segmentInfosFormat = new Lucene86SegmentInfoFormat();
- private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
- private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
- private final PointsFormat pointsFormat = new Lucene86PointsFormat();
- private final PostingsFormat defaultFormat;
-
- private final PostingsFormat postingsFormat =
- new PerFieldPostingsFormat() {
- @Override
- public PostingsFormat getPostingsFormatForField(String field) {
- return Lucene86Codec.this.getPostingsFormatForField(field);
- }
- };
-
- private final DocValuesFormat docValuesFormat =
- new PerFieldDocValuesFormat() {
- @Override
- public DocValuesFormat getDocValuesFormatForField(String field) {
- return Lucene86Codec.this.getDocValuesFormatForField(field);
- }
- };
-
- private final StoredFieldsFormat storedFieldsFormat;
-
- /** Instantiates a new codec. */
- public Lucene86Codec() {
- this(Lucene50StoredFieldsFormat.Mode.BEST_SPEED);
- }
-
- /**
- * Instantiates a new codec, specifying the stored fields compression mode to use.
- *
- * @param mode stored fields compression mode to use for newly flushed/merged segments.
- */
- public Lucene86Codec(Lucene50StoredFieldsFormat.Mode mode) {
- super("Lucene86");
- this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Objects.requireNonNull(mode));
- this.defaultFormat = new Lucene84PostingsFormat();
- }
-
- @Override
- public StoredFieldsFormat storedFieldsFormat() {
- return storedFieldsFormat;
- }
-
- @Override
- public TermVectorsFormat termVectorsFormat() {
- return vectorsFormat;
- }
-
- @Override
- public PostingsFormat postingsFormat() {
- return postingsFormat;
- }
-
- @Override
- public final FieldInfosFormat fieldInfosFormat() {
- return fieldInfosFormat;
- }
-
- @Override
- public SegmentInfoFormat segmentInfoFormat() {
- return segmentInfosFormat;
- }
-
- @Override
- public final LiveDocsFormat liveDocsFormat() {
- return liveDocsFormat;
- }
-
- @Override
- public CompoundFormat compoundFormat() {
- return compoundFormat;
- }
-
- @Override
- public PointsFormat pointsFormat() {
- return pointsFormat;
- }
-
- @Override
- public final KnnVectorsFormat knnVectorsFormat() {
- return KnnVectorsFormat.EMPTY;
- }
-
- /**
- * Returns the postings format that should be used for writing new segments of <code>field</code>.
- *
- * <p>The default implementation always returns "Lucene84".
- *
- * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
- * future version of Lucene are only guaranteed to be able to read the default implementation.
- */
- public PostingsFormat getPostingsFormatForField(String field) {
- return defaultFormat;
- }
-
- /**
- * Returns the docvalues format that should be used for writing new segments of <code>field</code>
- * .
- *
- * <p>The default implementation always returns "Lucene80".
- *
- * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
- * future version of Lucene are only guaranteed to be able to read the default implementation.
- */
- public DocValuesFormat getDocValuesFormatForField(String field) {
- return defaultDVFormat;
- }
-
- @Override
- public final DocValuesFormat docValuesFormat() {
- return docValuesFormat;
- }
-
- private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene80");
-
- private final NormsFormat normsFormat = new Lucene80NormsFormat();
-
- @Override
- public NormsFormat normsFormat() {
- return normsFormat;
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86PointsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86PointsFormat.java
deleted file mode 100644
index b8c4d8cd118..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86PointsFormat.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene86;
-
-import java.io.IOException;
-import org.apache.lucene.codecs.PointsFormat;
-import org.apache.lucene.codecs.PointsReader;
-import org.apache.lucene.codecs.PointsWriter;
-import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.index.SegmentWriteState;
-
-/**
- * Lucene 8.6 point format, which encodes dimensional values in a block KD-tree structure for fast
- * 1D range and N dimensional shape intersection filtering. See <a
- * href="https://www.cs.duke.edu/~pankaj/publications/papers/bkd-sstd.pdf">this paper</a> for
- * details.
- *
- * <p>Data is stored across three files
- *
- * <ul>
- * <li>A .kdm file that records metadata about the fields, such as numbers of dimensions or
- * numbers of bytes per dimension.
- * <li>A .kdi file that stores inner nodes of the tree.
- * <li>A .kdd file that stores leaf nodes, where most of the data lives.
- * </ul>
- *
- * See <a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=173081898">this
- * wiki</a> for detailed data structures of the three files.
- *
- * @lucene.experimental
- */
-public class Lucene86PointsFormat extends PointsFormat {
-
- static final String DATA_CODEC_NAME = "Lucene86PointsFormatData";
- static final String INDEX_CODEC_NAME = "Lucene86PointsFormatIndex";
- static final String META_CODEC_NAME = "Lucene86PointsFormatMeta";
-
- /** Filename extension for the leaf blocks */
- public static final String DATA_EXTENSION = "kdd";
-
- /** Filename extension for the index per field */
- public static final String INDEX_EXTENSION = "kdi";
-
- /** Filename extension for the meta per field */
- public static final String META_EXTENSION = "kdm";
-
- static final int VERSION_START = 0;
- static final int VERSION_CURRENT = VERSION_START;
-
- /** Sole constructor */
- public Lucene86PointsFormat() {}
-
- @Override
- public PointsWriter fieldsWriter(SegmentWriteState state) throws IOException {
- throw new UnsupportedOperationException("Old codecs may only be used for reading");
- }
-
- @Override
- public PointsReader fieldsReader(SegmentReadState state) throws IOException {
- return new Lucene86PointsReader(state);
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86PointsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86PointsReader.java
deleted file mode 100644
index efe6f5dccfe..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86PointsReader.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene86;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.PointsReader;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.PointValues;
-import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.bkd.BKDReader;
-
-/** Reads point values previously written with Lucene86PointsWriter */
-public class Lucene86PointsReader extends PointsReader {
- final IndexInput indexIn, dataIn;
- final SegmentReadState readState;
- final Map<Integer, PointValues> readers = new HashMap<>();
-
- /** Sole constructor */
- public Lucene86PointsReader(SegmentReadState readState) throws IOException {
- this.readState = readState;
-
- String metaFileName =
- IndexFileNames.segmentFileName(
- readState.segmentInfo.name,
- readState.segmentSuffix,
- Lucene86PointsFormat.META_EXTENSION);
- String indexFileName =
- IndexFileNames.segmentFileName(
- readState.segmentInfo.name,
- readState.segmentSuffix,
- Lucene86PointsFormat.INDEX_EXTENSION);
- String dataFileName =
- IndexFileNames.segmentFileName(
- readState.segmentInfo.name,
- readState.segmentSuffix,
- Lucene86PointsFormat.DATA_EXTENSION);
-
- boolean success = false;
- try {
- indexIn =
- EndiannessReverserUtil.openInput(readState.directory, indexFileName, readState.context);
- CodecUtil.checkIndexHeader(
- indexIn,
- Lucene86PointsFormat.INDEX_CODEC_NAME,
- Lucene86PointsFormat.VERSION_START,
- Lucene86PointsFormat.VERSION_CURRENT,
- readState.segmentInfo.getId(),
- readState.segmentSuffix);
-
- dataIn =
- EndiannessReverserUtil.openInput(readState.directory, dataFileName, readState.context);
- CodecUtil.checkIndexHeader(
- dataIn,
- Lucene86PointsFormat.DATA_CODEC_NAME,
- Lucene86PointsFormat.VERSION_START,
- Lucene86PointsFormat.VERSION_CURRENT,
- readState.segmentInfo.getId(),
- readState.segmentSuffix);
-
- long indexLength = -1, dataLength = -1;
- try (ChecksumIndexInput metaIn =
- EndiannessReverserUtil.openChecksumInput(
- readState.directory, metaFileName, readState.context)) {
- Throwable priorE = null;
- try {
- CodecUtil.checkIndexHeader(
- metaIn,
- Lucene86PointsFormat.META_CODEC_NAME,
- Lucene86PointsFormat.VERSION_START,
- Lucene86PointsFormat.VERSION_CURRENT,
- readState.segmentInfo.getId(),
- readState.segmentSuffix);
-
- while (true) {
- int fieldNumber = metaIn.readInt();
- if (fieldNumber == -1) {
- break;
- } else if (fieldNumber < 0) {
- throw new CorruptIndexException("Illegal field number: " + fieldNumber, metaIn);
- }
- PointValues reader = new BKDReader(metaIn, indexIn, dataIn);
- readers.put(fieldNumber, reader);
- }
- indexLength = metaIn.readLong();
- dataLength = metaIn.readLong();
- } catch (Throwable t) {
- priorE = t;
- } finally {
- CodecUtil.checkFooter(metaIn, priorE);
- }
- }
- // At this point, checksums of the meta file have been validated so we
- // know that indexLength and dataLength are very likely correct.
- CodecUtil.retrieveChecksum(indexIn, indexLength);
- CodecUtil.retrieveChecksum(dataIn, dataLength);
- success = true;
- } finally {
- if (success == false) {
- IOUtils.closeWhileHandlingException(this);
- }
- }
- }
-
- /**
- * Returns the underlying {@link PointValues}.
- *
- * @lucene.internal
- */
- @Override
- public PointValues getValues(String fieldName) {
- FieldInfo fieldInfo = readState.fieldInfos.fieldInfo(fieldName);
- if (fieldInfo == null) {
- throw new IllegalArgumentException("field=\"" + fieldName + "\" is unrecognized");
- }
- if (fieldInfo.getPointDimensionCount() == 0) {
- throw new IllegalArgumentException("field=\"" + fieldName + "\" did not index point values");
- }
-
- return readers.get(fieldInfo.number);
- }
-
- @Override
- public void checkIntegrity() throws IOException {
- CodecUtil.checksumEntireFile(indexIn);
- CodecUtil.checksumEntireFile(dataIn);
- }
-
- @Override
- public void close() throws IOException {
- IOUtils.close(indexIn, dataIn);
- // Free up heap:
- readers.clear();
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86SegmentInfoFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86SegmentInfoFormat.java
deleted file mode 100644
index b7b660ff4f5..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86SegmentInfoFormat.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.lucene.backward_codecs.lucene86;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.Set;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.SegmentInfoFormat;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.SegmentInfos;
-import org.apache.lucene.index.SortFieldProvider;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.Version;
-
-/**
- * Lucene 8.6 Segment info format.
- *
- * <p>Files:
- *
- * <ul>
- * <li><code>.si</code>: Header, SegVersion, SegSize, IsCompoundFile, Diagnostics, Files,
- * Attributes, IndexSort, Footer
- * </ul>
- *
- * Data types:
- *
- * <ul>
- * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
- * <li>SegSize --&gt; {@link DataOutput#writeInt Int32}
- * <li>SegVersion --&gt; {@link DataOutput#writeString String}
- * <li>SegMinVersion --&gt; {@link DataOutput#writeString String}
- * <li>Files --&gt; {@link DataOutput#writeSetOfStrings Set&lt;String&gt;}
- * <li>Diagnostics,Attributes --&gt; {@link DataOutput#writeMapOfStrings Map&lt;String,String&gt;}
- * <li>IsCompoundFile --&gt; {@link DataOutput#writeByte Int8}
- * <li>IndexSort --&gt; {@link DataOutput#writeVInt Int32} count, followed by {@code count}
- * SortField
- * <li>SortField --&gt; {@link DataOutput#writeString String} sort class, followed by a per-sort
- * bytestream (see {@link SortFieldProvider#readSortField(DataInput)})
- * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
- * </ul>
- *
- * Field Descriptions:
- *
- * <ul>
- * <li>SegVersion is the code version that created the segment.
- * <li>SegMinVersion is the minimum code version that contributed documents to the segment.
- * <li>SegSize is the number of documents contained in the segment index.
- * <li>IsCompoundFile records whether the segment is written as a compound file or not. If this is
- * -1, the segment is not a compound file. If it is 1, the segment is a compound file.
- * <li>The Diagnostics Map is privately written by {@link IndexWriter}, as a debugging aid, for
- * each segment it creates. It includes metadata like the current Lucene version, OS, Java
- * version, why the segment was created (merge, flush, addIndexes), etc.
- * <li>Files is a list of files referred to by this segment.
- * </ul>
- *
- * @see SegmentInfos
- * @lucene.experimental
- */
-public class Lucene86SegmentInfoFormat extends SegmentInfoFormat {
-
- /** File extension used to store {@link SegmentInfo}. */
- public static final String SI_EXTENSION = "si";
-
- static final String CODEC_NAME = "Lucene86SegmentInfo";
- static final int VERSION_START = 0;
- static final int VERSION_CURRENT = VERSION_START;
-
- /** Sole constructor. */
- public Lucene86SegmentInfoFormat() {}
-
- @Override
- public SegmentInfo read(Directory dir, String segment, byte[] segmentID, IOContext context)
- throws IOException {
- final String fileName = IndexFileNames.segmentFileName(segment, "", SI_EXTENSION);
- try (ChecksumIndexInput input =
- EndiannessReverserUtil.openChecksumInput(dir, fileName, context)) {
- Throwable priorE = null;
- SegmentInfo si = null;
- try {
- CodecUtil.checkIndexHeader(
- input, CODEC_NAME, VERSION_START, VERSION_CURRENT, segmentID, "");
- si = parseSegmentInfo(dir, input, segment, segmentID);
- } catch (Throwable exception) {
- priorE = exception;
- } finally {
- CodecUtil.checkFooter(input, priorE);
- }
- return si;
- }
- }
-
- private SegmentInfo parseSegmentInfo(
- Directory dir, DataInput input, String segment, byte[] segmentID) throws IOException {
- final Version version = Version.fromBits(input.readInt(), input.readInt(), input.readInt());
- byte hasMinVersion = input.readByte();
- final Version minVersion;
- switch (hasMinVersion) {
- case 0:
- minVersion = null;
- break;
- case 1:
- minVersion = Version.fromBits(input.readInt(), input.readInt(), input.readInt());
- break;
- default:
- throw new CorruptIndexException("Illegal boolean value " + hasMinVersion, input);
- }
-
- final int docCount = input.readInt();
- if (docCount < 0) {
- throw new CorruptIndexException("invalid docCount: " + docCount, input);
- }
- final boolean isCompoundFile = input.readByte() == SegmentInfo.YES;
-
- final Map<String, String> diagnostics = input.readMapOfStrings();
- final Set<String> files = input.readSetOfStrings();
- final Map<String, String> attributes = input.readMapOfStrings();
-
- int numSortFields = input.readVInt();
- Sort indexSort;
- if (numSortFields > 0) {
- SortField[] sortFields = new SortField[numSortFields];
- for (int i = 0; i < numSortFields; i++) {
- String name = input.readString();
- sortFields[i] = SortFieldProvider.forName(name).readSortField(input);
- }
- indexSort = new Sort(sortFields);
- } else if (numSortFields < 0) {
- throw new CorruptIndexException("invalid index sort field count: " + numSortFields, input);
- } else {
- indexSort = null;
- }
-
- SegmentInfo si =
- new SegmentInfo(
- dir,
- version,
- minVersion,
- segment,
- docCount,
- isCompoundFile,
- false,
- null,
- diagnostics,
- segmentID,
- attributes,
- indexSort);
- si.setFiles(files);
- return si;
- }
-
- @Override
- public void write(Directory dir, SegmentInfo si, IOContext ioContext) throws IOException {
- throw new UnsupportedOperationException("Old formats can't be used for writing");
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/package-info.java
deleted file mode 100644
index c516fa3ce69..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** Lucene 8.6 file format. */
-package org.apache.lucene.backward_codecs.lucene86;
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/DeflateWithPresetDictCompressionMode.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/DeflateWithPresetDictCompressionMode.java
deleted file mode 100644
index 8ff41e64a05..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/DeflateWithPresetDictCompressionMode.java
+++ /dev/null
@@ -1,228 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene87;
-
-import java.io.IOException;
-import java.util.zip.DataFormatException;
-import java.util.zip.Deflater;
-import java.util.zip.Inflater;
-import org.apache.lucene.backward_codecs.compressing.CompressionMode;
-import org.apache.lucene.backward_codecs.compressing.Compressor;
-import org.apache.lucene.backward_codecs.compressing.Decompressor;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-
-/**
- * A compression mode that trades speed for compression ratio. Although compression and
- * decompression might be slow, this compression mode should provide a good compression ratio. This
- * mode might be interesting if/when your index size is much bigger than your OS cache.
- *
- * @lucene.internal
- */
-public final class DeflateWithPresetDictCompressionMode extends CompressionMode {
-
- // Shoot for 10 sub blocks
- private static final int NUM_SUB_BLOCKS = 10;
- // And a dictionary whose size is about 6x smaller than sub blocks
- private static final int DICT_SIZE_FACTOR = 6;
-
- /** Sole constructor. */
- public DeflateWithPresetDictCompressionMode() {}
-
- @Override
- public Compressor newCompressor() {
- // notes:
- // 3 is the highest level that doesn't have lazy match evaluation
- // 6 is the default, higher than that is just a waste of cpu
- return new DeflateWithPresetDictCompressor(6);
- }
-
- @Override
- public Decompressor newDecompressor() {
- return new DeflateWithPresetDictDecompressor();
- }
-
- @Override
- public String toString() {
- return "BEST_COMPRESSION";
- }
-
- private static final class DeflateWithPresetDictDecompressor extends Decompressor {
-
- byte[] compressed;
-
- DeflateWithPresetDictDecompressor() {
- compressed = new byte[0];
- }
-
- private void doDecompress(DataInput in, Inflater decompressor, BytesRef bytes)
- throws IOException {
- final int compressedLength = in.readVInt();
- if (compressedLength == 0) {
- return;
- }
- // pad with extra "dummy byte": see javadocs for using Inflater(true)
- // we do it for compliance, but it's unnecessary for years in zlib.
- final int paddedLength = compressedLength + 1;
- compressed = ArrayUtil.grow(compressed, paddedLength);
- in.readBytes(compressed, 0, compressedLength);
- compressed[compressedLength] = 0; // explicitly set dummy byte to 0
-
- // extra "dummy byte"
- decompressor.setInput(compressed, 0, paddedLength);
- try {
- bytes.length +=
- decompressor.inflate(bytes.bytes, bytes.length, bytes.bytes.length - bytes.length);
- } catch (DataFormatException e) {
- throw new IOException(e);
- }
- if (decompressor.finished() == false) {
- throw new CorruptIndexException(
- "Invalid decoder state: needsInput="
- + decompressor.needsInput()
- + ", needsDict="
- + decompressor.needsDictionary(),
- in);
- }
- }
-
- @Override
- public void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes)
- throws IOException {
- assert offset + length <= originalLength;
- if (length == 0) {
- bytes.length = 0;
- return;
- }
- final int dictLength = in.readVInt();
- final int blockLength = in.readVInt();
- bytes.bytes = ArrayUtil.grow(bytes.bytes, dictLength);
- bytes.offset = bytes.length = 0;
-
- final Inflater decompressor = new Inflater(true);
- try {
- // Read the dictionary
- doDecompress(in, decompressor, bytes);
- if (dictLength != bytes.length) {
- throw new CorruptIndexException("Unexpected dict length", in);
- }
-
- int offsetInBlock = dictLength;
- int offsetInBytesRef = offset;
-
- // Skip unneeded blocks
- while (offsetInBlock + blockLength < offset) {
- final int compressedLength = in.readVInt();
- in.skipBytes(compressedLength);
- offsetInBlock += blockLength;
- offsetInBytesRef -= blockLength;
- }
-
- // Read blocks that intersect with the interval we need
- while (offsetInBlock < offset + length) {
- bytes.bytes = ArrayUtil.grow(bytes.bytes, bytes.length + blockLength);
- decompressor.reset();
- decompressor.setDictionary(bytes.bytes, 0, dictLength);
- doDecompress(in, decompressor, bytes);
- offsetInBlock += blockLength;
- }
-
- bytes.offset = offsetInBytesRef;
- bytes.length = length;
- assert bytes.isValid();
- } finally {
- decompressor.end();
- }
- }
-
- @Override
- public Decompressor clone() {
- return new DeflateWithPresetDictDecompressor();
- }
- }
-
- private static class DeflateWithPresetDictCompressor extends Compressor {
-
- final Deflater compressor;
- byte[] compressed;
- boolean closed;
-
- DeflateWithPresetDictCompressor(int level) {
- compressor = new Deflater(level, true);
- compressed = new byte[64];
- }
-
- private void doCompress(byte[] bytes, int off, int len, DataOutput out) throws IOException {
- if (len == 0) {
- out.writeVInt(0);
- return;
- }
- compressor.setInput(bytes, off, len);
- compressor.finish();
- if (compressor.needsInput()) {
- throw new IllegalStateException();
- }
-
- int totalCount = 0;
- for (; ; ) {
- final int count =
- compressor.deflate(compressed, totalCount, compressed.length - totalCount);
- totalCount += count;
- assert totalCount <= compressed.length;
- if (compressor.finished()) {
- break;
- } else {
- compressed = ArrayUtil.grow(compressed);
- }
- }
-
- out.writeVInt(totalCount);
- out.writeBytes(compressed, totalCount);
- }
-
- @Override
- public void compress(byte[] bytes, int off, int len, DataOutput out) throws IOException {
- final int dictLength = len / (NUM_SUB_BLOCKS * DICT_SIZE_FACTOR);
- final int blockLength = (len - dictLength + NUM_SUB_BLOCKS - 1) / NUM_SUB_BLOCKS;
- out.writeVInt(dictLength);
- out.writeVInt(blockLength);
- final int end = off + len;
-
- // Compress the dictionary first
- compressor.reset();
- doCompress(bytes, off, dictLength, out);
-
- // And then sub blocks
- for (int start = off + dictLength; start < end; start += blockLength) {
- compressor.reset();
- compressor.setDictionary(bytes, off, dictLength);
- doCompress(bytes, start, Math.min(blockLength, off + len - start), out);
- }
- }
-
- @Override
- public void close() throws IOException {
- if (closed == false) {
- compressor.end();
- closed = true;
- }
- }
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/LZ4WithPresetDictCompressionMode.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/LZ4WithPresetDictCompressionMode.java
deleted file mode 100644
index e8d40308d0c..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/LZ4WithPresetDictCompressionMode.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene87;
-
-import java.io.IOException;
-import org.apache.lucene.backward_codecs.compressing.CompressionMode;
-import org.apache.lucene.backward_codecs.compressing.Compressor;
-import org.apache.lucene.backward_codecs.compressing.Decompressor;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.store.ByteBuffersDataOutput;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.compress.LZ4;
-
-/**
- * A compression mode that compromises on the compression ratio to provide fast compression and
- * decompression.
- *
- * @lucene.internal
- */
-public final class LZ4WithPresetDictCompressionMode extends CompressionMode {
-
- // Shoot for 10 sub blocks
- private static final int NUM_SUB_BLOCKS = 10;
- // And a dictionary whose size is about 2x smaller than sub blocks
- private static final int DICT_SIZE_FACTOR = 2;
-
- /** Sole constructor. */
- public LZ4WithPresetDictCompressionMode() {}
-
- @Override
- public Compressor newCompressor() {
- return new LZ4WithPresetDictCompressor();
- }
-
- @Override
- public Decompressor newDecompressor() {
- return new LZ4WithPresetDictDecompressor();
- }
-
- @Override
- public String toString() {
- return "BEST_SPEED";
- }
-
- private static final class LZ4WithPresetDictDecompressor extends Decompressor {
-
- private int[] compressedLengths;
- private byte[] buffer;
-
- LZ4WithPresetDictDecompressor() {
- compressedLengths = new int[0];
- buffer = new byte[0];
- }
-
- private int readCompressedLengths(
- DataInput in, int originalLength, int dictLength, int blockLength) throws IOException {
- in.readVInt(); // compressed length of the dictionary, unused
- int totalLength = dictLength;
- int i = 0;
- while (totalLength < originalLength) {
- compressedLengths = ArrayUtil.grow(compressedLengths, i + 1);
- compressedLengths[i++] = in.readVInt();
- totalLength += blockLength;
- }
- return i;
- }
-
- @Override
- public void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes)
- throws IOException {
- assert offset + length <= originalLength;
-
- if (length == 0) {
- bytes.length = 0;
- return;
- }
-
- final int dictLength = in.readVInt();
- final int blockLength = in.readVInt();
-
- final int numBlocks = readCompressedLengths(in, originalLength, dictLength, blockLength);
-
- buffer = ArrayUtil.grow(buffer, dictLength + blockLength);
- bytes.length = 0;
- // Read the dictionary
- if (LZ4.decompress(EndiannessReverserUtil.wrapDataInput(in), dictLength, buffer, 0)
- != dictLength) {
- throw new CorruptIndexException("Illegal dict length", in);
- }
-
- int offsetInBlock = dictLength;
- int offsetInBytesRef = offset;
- if (offset >= dictLength) {
- offsetInBytesRef -= dictLength;
-
- // Skip unneeded blocks
- int numBytesToSkip = 0;
- for (int i = 0; i < numBlocks && offsetInBlock + blockLength < offset; ++i) {
- int compressedBlockLength = compressedLengths[i];
- numBytesToSkip += compressedBlockLength;
- offsetInBlock += blockLength;
- offsetInBytesRef -= blockLength;
- }
- in.skipBytes(numBytesToSkip);
- } else {
- // The dictionary contains some bytes we need, copy its content to the BytesRef
- bytes.bytes = ArrayUtil.grow(bytes.bytes, dictLength);
- System.arraycopy(buffer, 0, bytes.bytes, 0, dictLength);
- bytes.length = dictLength;
- }
-
- // Read blocks that intersect with the interval we need
- while (offsetInBlock < offset + length) {
- final int bytesToDecompress = Math.min(blockLength, offset + length - offsetInBlock);
- LZ4.decompress(
- EndiannessReverserUtil.wrapDataInput(in), bytesToDecompress, buffer, dictLength);
- bytes.bytes = ArrayUtil.grow(bytes.bytes, bytes.length + bytesToDecompress);
- System.arraycopy(buffer, dictLength, bytes.bytes, bytes.length, bytesToDecompress);
- bytes.length += bytesToDecompress;
- offsetInBlock += blockLength;
- }
-
- bytes.offset = offsetInBytesRef;
- bytes.length = length;
- assert bytes.isValid();
- }
-
- @Override
- public Decompressor clone() {
- return new LZ4WithPresetDictDecompressor();
- }
- }
-
- private static class LZ4WithPresetDictCompressor extends Compressor {
-
- final ByteBuffersDataOutput compressed;
- final LZ4.FastCompressionHashTable hashTable;
- byte[] buffer;
-
- LZ4WithPresetDictCompressor() {
- compressed = ByteBuffersDataOutput.newResettableInstance();
- hashTable = new LZ4.FastCompressionHashTable();
- buffer = BytesRef.EMPTY_BYTES;
- }
-
- private void doCompress(byte[] bytes, int dictLen, int len, DataOutput out) throws IOException {
- long prevCompressedSize = compressed.size();
- LZ4.compressWithDictionary(bytes, 0, dictLen, len, compressed, hashTable);
- // Write the number of compressed bytes
- out.writeVInt(Math.toIntExact(compressed.size() - prevCompressedSize));
- }
-
- @Override
- public void compress(byte[] bytes, int off, int len, DataOutput out) throws IOException {
- final int dictLength = len / (NUM_SUB_BLOCKS * DICT_SIZE_FACTOR);
- final int blockLength = (len - dictLength + NUM_SUB_BLOCKS - 1) / NUM_SUB_BLOCKS;
- buffer = ArrayUtil.grow(buffer, dictLength + blockLength);
- out.writeVInt(dictLength);
- out.writeVInt(blockLength);
- final int end = off + len;
-
- compressed.reset();
- // Compress the dictionary first
- System.arraycopy(bytes, off, buffer, 0, dictLength);
- doCompress(buffer, 0, dictLength, out);
-
- // And then sub blocks
- for (int start = off + dictLength; start < end; start += blockLength) {
- int l = Math.min(blockLength, off + len - start);
- System.arraycopy(bytes, start, buffer, dictLength, l);
- doCompress(buffer, dictLength, l, out);
- }
-
- // We only wrote lengths so far, now write compressed data
- compressed.copyTo(out);
- }
-
- @Override
- public void close() throws IOException {
- // no-op
- }
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/Lucene87Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/Lucene87Codec.java
deleted file mode 100644
index 53a235ab082..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/Lucene87Codec.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.lucene.backward_codecs.lucene87;
-
-import java.util.Objects;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat;
-import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;
-import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat;
-import org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat;
-import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat;
-import org.apache.lucene.backward_codecs.lucene86.Lucene86PointsFormat;
-import org.apache.lucene.backward_codecs.lucene86.Lucene86SegmentInfoFormat;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.CompoundFormat;
-import org.apache.lucene.codecs.DocValuesFormat;
-import org.apache.lucene.codecs.FieldInfosFormat;
-import org.apache.lucene.codecs.FilterCodec;
-import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.LiveDocsFormat;
-import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.PointsFormat;
-import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.SegmentInfoFormat;
-import org.apache.lucene.codecs.StoredFieldsFormat;
-import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
-import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
-
-/**
- * Implements the Lucene 8.6 index format, with configurable per-field postings and docvalues
- * formats.
- *
- * <p>If you want to reuse functionality of this codec in another codec, extend {@link FilterCodec}.
- *
- * @lucene.experimental
- */
-public class Lucene87Codec extends Codec {
-
- /** Configuration option for the codec. */
- public enum Mode {
- /** Trade compression ratio for retrieval speed. */
- BEST_SPEED(Lucene87StoredFieldsFormat.Mode.BEST_SPEED, Lucene80DocValuesFormat.Mode.BEST_SPEED),
- /** Trade retrieval speed for compression ratio. */
- BEST_COMPRESSION(
- Lucene87StoredFieldsFormat.Mode.BEST_COMPRESSION,
- Lucene80DocValuesFormat.Mode.BEST_COMPRESSION);
-
- /** compression mode for stored fields */
- protected final Lucene87StoredFieldsFormat.Mode storedMode;
-
- /** compression mode for doc value fields */
- protected final Lucene80DocValuesFormat.Mode dvMode;
-
- private Mode(Lucene87StoredFieldsFormat.Mode storedMode, Lucene80DocValuesFormat.Mode dvMode) {
- this.storedMode = Objects.requireNonNull(storedMode);
- this.dvMode = Objects.requireNonNull(dvMode);
- }
- }
-
- private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat();
- private final FieldInfosFormat fieldInfosFormat = new Lucene60FieldInfosFormat();
- private final SegmentInfoFormat segmentInfosFormat = new Lucene86SegmentInfoFormat();
- private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
- private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
- private final PointsFormat pointsFormat = new Lucene86PointsFormat();
- private final PostingsFormat defaultFormat;
-
- private final PostingsFormat postingsFormat =
- new PerFieldPostingsFormat() {
- @Override
- public PostingsFormat getPostingsFormatForField(String field) {
- return Lucene87Codec.this.getPostingsFormatForField(field);
- }
- };
-
- private final DocValuesFormat docValuesFormat =
- new PerFieldDocValuesFormat() {
- @Override
- public DocValuesFormat getDocValuesFormatForField(String field) {
- return Lucene87Codec.this.getDocValuesFormatForField(field);
- }
- };
-
- private final StoredFieldsFormat storedFieldsFormat;
-
- /** Instantiates a new codec. */
- public Lucene87Codec() {
- this(Mode.BEST_COMPRESSION);
- }
-
- /**
- * Instantiates a new codec, specifying the compression mode to use.
- *
- * @param mode compression mode to use for newly flushed/merged segments.
- */
- public Lucene87Codec(Mode mode) {
- super("Lucene87");
- this.storedFieldsFormat = new Lucene87StoredFieldsFormat(mode.storedMode);
- this.defaultFormat = new Lucene84PostingsFormat();
- this.defaultDVFormat = new Lucene80DocValuesFormat(mode.dvMode);
- }
-
- @Override
- public StoredFieldsFormat storedFieldsFormat() {
- return storedFieldsFormat;
- }
-
- @Override
- public TermVectorsFormat termVectorsFormat() {
- return vectorsFormat;
- }
-
- @Override
- public PostingsFormat postingsFormat() {
- return postingsFormat;
- }
-
- @Override
- public final FieldInfosFormat fieldInfosFormat() {
- return fieldInfosFormat;
- }
-
- @Override
- public SegmentInfoFormat segmentInfoFormat() {
- return segmentInfosFormat;
- }
-
- @Override
- public final LiveDocsFormat liveDocsFormat() {
- return liveDocsFormat;
- }
-
- @Override
- public CompoundFormat compoundFormat() {
- return compoundFormat;
- }
-
- @Override
- public PointsFormat pointsFormat() {
- return pointsFormat;
- }
-
- @Override
- public final KnnVectorsFormat knnVectorsFormat() {
- return KnnVectorsFormat.EMPTY;
- }
-
- /**
- * Returns the postings format that should be used for writing new segments of <code>field</code>.
- *
- * <p>The default implementation always returns "Lucene84".
- *
- * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
- * future version of Lucene are only guaranteed to be able to read the default implementation.
- */
- public PostingsFormat getPostingsFormatForField(String field) {
- return defaultFormat;
- }
-
- /**
- * Returns the docvalues format that should be used for writing new segments of <code>field</code>
- * .
- *
- * <p>The default implementation always returns "Lucene80".
- *
- * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
- * future version of Lucene are only guaranteed to be able to read the default implementation.
- */
- public DocValuesFormat getDocValuesFormatForField(String field) {
- return defaultDVFormat;
- }
-
- @Override
- public final DocValuesFormat docValuesFormat() {
- return docValuesFormat;
- }
-
- private final DocValuesFormat defaultDVFormat;
-
- private final NormsFormat normsFormat = new Lucene80NormsFormat();
-
- @Override
- public NormsFormat normsFormat() {
- return normsFormat;
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/Lucene87StoredFieldsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/Lucene87StoredFieldsFormat.java
deleted file mode 100644
index 988b979354f..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/Lucene87StoredFieldsFormat.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene87;
-
-import java.io.IOException;
-import java.util.Objects;
-import org.apache.lucene.backward_codecs.compressing.CompressionMode;
-import org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsFormat;
-import org.apache.lucene.codecs.StoredFieldsFormat;
-import org.apache.lucene.codecs.StoredFieldsReader;
-import org.apache.lucene.codecs.StoredFieldsWriter;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.StoredFieldVisitor;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.packed.DirectMonotonicWriter;
-
-/**
- * Lucene 8.7 stored fields format.
- *
- * <p><b>Principle</b>
- *
- * <p>This {@link StoredFieldsFormat} compresses blocks of documents in order to improve the
- * compression ratio compared to document-level compression. It uses the <a
- * href="http://code.google.com/p/lz4/">LZ4</a> compression algorithm by default in 16KB blocks,
- * which is fast to compress and very fast to decompress data. Although the default compression
- * method that is used ({@link Mode#BEST_SPEED BEST_SPEED}) focuses more on speed than on
- * compression ratio, it should provide interesting compression ratios for redundant inputs (such as
- * log files, HTML or plain text). For higher compression, you can choose ({@link
- * Mode#BEST_COMPRESSION BEST_COMPRESSION}), which uses the <a
- * href="http://en.wikipedia.org/wiki/DEFLATE">DEFLATE</a> algorithm with 48kB blocks and shared
- * dictionaries for a better ratio at the expense of slower performance. These two options can be
- * configured like this:
- *
- * <pre><code class="language-java">
- * // the default: for high performance
- * indexWriterConfig.setCodec(new Lucene87Codec(Mode.BEST_SPEED));
- * // instead for higher performance (but slower):
- * // indexWriterConfig.setCodec(new Lucene87Codec(Mode.BEST_COMPRESSION));
- * </code></pre>
- *
- * <p><b>File formats</b>
- *
- * <p>Stored fields are represented by three files:
- *
- * <ol>
- * <li><a id="field_data"></a>
- * <p>A fields data file (extension <code>.fdt</code>). This file stores a compact
- * representation of documents in compressed blocks of 16KB or more. When writing a segment,
- * documents are appended to an in-memory <code>byte[]</code> buffer. When its size reaches
- * 16KB or more, some metadata about the documents is flushed to disk, immediately followed by
- * a compressed representation of the buffer using the <a
- * href="https://github.com/lz4/lz4">LZ4</a> <a
- * href="http://fastcompression.blogspot.fr/2011/05/lz4-explained.html">compression
- * format</a>.
- * <p>Notes
- * <ul>
- * <li>When at least one document in a chunk is large enough so that the chunk is larger
- * than 32KB, the chunk will actually be compressed in several LZ4 blocks of 16KB. This
- * allows {@link StoredFieldVisitor}s which are only interested in the first fields of a
- * document to not have to decompress 10MB of data if the document is 10MB, but only
- * 16KB.
- * <li>Given that the original lengths are written in the metadata of the chunk, the
- * decompressor can leverage this information to stop decoding as soon as enough data
- * has been decompressed.
- * <li>In case documents are incompressible, the overhead of the compression format is less
- * than 0.5%.
- * </ul>
- * <li><a id="field_index"></a>
- * <p>A fields index file (extension <code>.fdx</code>). This file stores two {@link
- * DirectMonotonicWriter monotonic arrays}, one for the first doc IDs of each block of
- * compressed documents, and another one for the corresponding offsets on disk. At search
- * time, the array containing doc IDs is binary-searched in order to find the block that
- * contains the expected doc ID, and the associated offset on disk is retrieved from the
- * second array.
- * <li><a id="field_meta"></a>
- * <p>A fields meta file (extension <code>.fdm</code>). This file stores metadata about the
- * monotonic arrays stored in the index file.
- * </ol>
- *
- * <p><b>Known limitations</b>
- *
- * <p>This {@link StoredFieldsFormat} does not support individual documents larger than (<code>
- * 2<sup>31</sup> - 2<sup>14</sup></code>) bytes.
- *
- * @lucene.experimental
- */
-public class Lucene87StoredFieldsFormat extends StoredFieldsFormat {
-
- /** Configuration option for stored fields. */
- public enum Mode {
- /** Trade compression ratio for retrieval speed. */
- BEST_SPEED,
- /** Trade retrieval speed for compression ratio. */
- BEST_COMPRESSION
- }
-
- /** Attribute key for compression mode. */
- public static final String MODE_KEY = Lucene87StoredFieldsFormat.class.getSimpleName() + ".mode";
-
- final Mode mode;
-
- /** Stored fields format with default options */
- public Lucene87StoredFieldsFormat() {
- this(Mode.BEST_SPEED);
- }
-
- /** Stored fields format with specified mode */
- public Lucene87StoredFieldsFormat(Mode mode) {
- this.mode = Objects.requireNonNull(mode);
- }
-
- @Override
- public StoredFieldsReader fieldsReader(
- Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
- String value = si.getAttribute(MODE_KEY);
- if (value == null) {
- throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name);
- }
- Mode mode = Mode.valueOf(value);
- return impl(mode).fieldsReader(directory, si, fn, context);
- }
-
- @Override
- public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context)
- throws IOException {
- throw new UnsupportedOperationException("Old codecs may only be used for reading");
- }
-
- StoredFieldsFormat impl(Mode mode) {
- switch (mode) {
- case BEST_SPEED:
- return new Lucene50CompressingStoredFieldsFormat(
- "Lucene87StoredFieldsFastData", BEST_SPEED_MODE, BEST_SPEED_BLOCK_LENGTH, 1024, 10);
- case BEST_COMPRESSION:
- return new Lucene50CompressingStoredFieldsFormat(
- "Lucene87StoredFieldsHighData",
- BEST_COMPRESSION_MODE,
- BEST_COMPRESSION_BLOCK_LENGTH,
- 4096,
- 10);
- default:
- throw new AssertionError();
- }
- }
-
- // Shoot for 10 sub blocks of 48kB each.
- /** Block length for {@link Mode#BEST_COMPRESSION} */
- protected static final int BEST_COMPRESSION_BLOCK_LENGTH = 10 * 48 * 1024;
-
- /** Compression mode for {@link Mode#BEST_COMPRESSION} */
- public static final CompressionMode BEST_COMPRESSION_MODE =
- new DeflateWithPresetDictCompressionMode();
-
- // Shoot for 10 sub blocks of 8kB each.
- /** Block length for {@link Mode#BEST_SPEED} */
- protected static final int BEST_SPEED_BLOCK_LENGTH = 10 * 8 * 1024;
-
- /** Compression mode for {@link Mode#BEST_SPEED} */
- public static final CompressionMode BEST_SPEED_MODE = new LZ4WithPresetDictCompressionMode();
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/package-info.java
deleted file mode 100644
index a77730b1b9e..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** Lucene 8.7 file format. */
-package org.apache.lucene.backward_codecs.lucene87;
diff --git a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
index 15bb6e9fa56..9b7ca1e5149 100644
--- a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
+++ b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
@@ -13,10 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.lucene.backward_codecs.lucene80.Lucene80Codec
-org.apache.lucene.backward_codecs.lucene84.Lucene84Codec
-org.apache.lucene.backward_codecs.lucene86.Lucene86Codec
-org.apache.lucene.backward_codecs.lucene87.Lucene87Codec
org.apache.lucene.backward_codecs.lucene90.Lucene90Codec
org.apache.lucene.backward_codecs.lucene91.Lucene91Codec
org.apache.lucene.backward_codecs.lucene92.Lucene92Codec
diff --git a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
index a0c1266b755..87c5b6739a4 100644
--- a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
+++ b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
@@ -13,8 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat
-org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat
org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat
org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat
org.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormat
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene40/blocktree/Lucene40BlockTreeTermsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene40/blocktree/Lucene40BlockTreeTermsWriter.java
deleted file mode 100644
index 6bcc0671a5c..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene40/blocktree/Lucene40BlockTreeTermsWriter.java
+++ /dev/null
@@ -1,1164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene40.blocktree;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Objects;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.BlockTermState;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.FieldsConsumer;
-import org.apache.lucene.codecs.NormsProducer;
-import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.Fields;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.store.ByteArrayDataOutput;
-import org.apache.lucene.store.ByteBuffersDataOutput;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.FixedBitSet;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.IntsRefBuilder;
-import org.apache.lucene.util.StringHelper;
-import org.apache.lucene.util.ToStringUtils;
-import org.apache.lucene.util.compress.LZ4;
-import org.apache.lucene.util.compress.LowercaseAsciiCompression;
-import org.apache.lucene.util.fst.ByteSequenceOutputs;
-import org.apache.lucene.util.fst.BytesRefFSTEnum;
-import org.apache.lucene.util.fst.FST;
-import org.apache.lucene.util.fst.FSTCompiler;
-import org.apache.lucene.util.fst.Util;
-
-/*
- TODO:
-
- - Currently there is a one-to-one mapping of indexed
- term to term block, but we could decouple the two, ie,
- put more terms into the index than there are blocks.
- The index would take up more RAM but then it'd be able
- to avoid seeking more often and could make PK/FuzzyQ
- faster if the additional indexed terms could store
- the offset into the terms block.
-
- - The blocks are not written in true depth-first
- order, meaning if you just next() the file pointer will
- sometimes jump backwards. For example, block foo* will
- be written before block f* because it finished before.
- This could possibly hurt performance if the terms dict is
- not hot, since OSs anticipate sequential file access. We
- could fix the writer to re-order the blocks as a 2nd
- pass.
-
- - Each block encodes the term suffixes packed
- sequentially using a separate vInt per term, which is
- 1) wasteful and 2) slow (must linear scan to find a
- particular suffix). We should instead 1) make
- random-access array so we can directly access the Nth
- suffix, and 2) bulk-encode this array using bulk int[]
- codecs; then at search time we can binary search when
- we seek a particular term.
-*/
-
-/**
- * Block-based terms index and dictionary writer.
- *
- * <p>Writes terms dict and index, block-encoding (column stride) each term's metadata for each set
- * of terms between two index terms.
- *
- * <p>Files:
- *
- * <ul>
- * <li><code>.tim</code>: <a href="#Termdictionary">Term Dictionary</a>
- * <li><code>.tip</code>: <a href="#Termindex">Term Index</a>
- * </ul>
- *
- * <p><a id="Termdictionary"></a>
- *
- * <h2>Term Dictionary</h2>
- *
- * <p>The .tim file contains the list of terms in each field along with per-term statistics (such as
- * docfreq) and per-term metadata (typically pointers to the postings list for that term in the
- * inverted index).
- *
- * <p>The .tim is arranged in blocks: with blocks containing a variable number of entries (by
- * default 25-48), where each entry is either a term or a reference to a sub-block.
- *
- * <p>NOTE: The term dictionary can plug into different postings implementations: the postings
- * writer/reader are actually responsible for encoding and decoding the Postings Metadata and Term
- * Metadata sections.
- *
- * <ul>
- * <li>TermsDict (.tim) --&gt; Header, <i>PostingsHeader</i>, NodeBlock<sup>NumBlocks</sup>,
- * FieldSummary, DirOffset, Footer
- * <li>NodeBlock --&gt; (OuterNode | InnerNode)
- * <li>OuterNode --&gt; EntryCount, SuffixLength, Byte<sup>SuffixLength</sup>, StatsLength, &lt;
- * TermStats &gt;<sup>EntryCount</sup>, MetaLength,
- * &lt;<i>TermMetadata</i>&gt;<sup>EntryCount</sup>
- * <li>InnerNode --&gt; EntryCount, SuffixLength[,Sub?], Byte<sup>SuffixLength</sup>, StatsLength,
- * &lt; TermStats ? &gt;<sup>EntryCount</sup>, MetaLength, &lt;<i>TermMetadata ?
- * </i>&gt;<sup>EntryCount</sup>
- * <li>TermStats --&gt; DocFreq, TotalTermFreq
- * <li>FieldSummary --&gt; NumFields, &lt;FieldNumber, NumTerms, RootCodeLength,
- * Byte<sup>RootCodeLength</sup>, SumTotalTermFreq?, SumDocFreq, DocCount, LongsSize, MinTerm,
- * MaxTerm&gt;<sup>NumFields</sup>
- * <li>Header --&gt; {@link CodecUtil#writeHeader CodecHeader}
- * <li>DirOffset --&gt; {@link DataOutput#writeLong Uint64}
- * <li>MinTerm,MaxTerm --&gt; {@link DataOutput#writeVInt VInt} length followed by the byte[]
- * <li>EntryCount,SuffixLength,StatsLength,DocFreq,MetaLength,NumFields,
- * FieldNumber,RootCodeLength,DocCount,LongsSize --&gt; {@link DataOutput#writeVInt VInt}
- * <li>TotalTermFreq,NumTerms,SumTotalTermFreq,SumDocFreq --&gt; {@link DataOutput#writeVLong
- * VLong}
- * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
- * </ul>
- *
- * <p>Notes:
- *
- * <ul>
- * <li>Header is a {@link CodecUtil#writeHeader CodecHeader} storing the version information for
- * the BlockTree implementation.
- * <li>DirOffset is a pointer to the FieldSummary section.
- * <li>DocFreq is the count of documents which contain the term.
- * <li>TotalTermFreq is the total number of occurrences of the term. This is encoded as the
- * difference between the total number of occurrences and the DocFreq.
- * <li>FieldNumber is the fields number from {@link FieldInfos}. (.fnm)
- * <li>NumTerms is the number of unique terms for the field.
- * <li>RootCode points to the root block for the field.
- * <li>SumDocFreq is the total number of postings, the number of term-document pairs across the
- * entire field.
- * <li>DocCount is the number of documents that have at least one posting for this field.
- * <li>LongsSize records how many long values the postings writer/reader record per term (e.g., to
- * hold freq/prox/doc file offsets).
- * <li>MinTerm, MaxTerm are the lowest and highest term in this field.
- * <li>PostingsHeader and TermMetadata are plugged into by the specific postings implementation:
- * these contain arbitrary per-file data (such as parameters or versioning information) and
- * per-term data (such as pointers to inverted files).
- * <li>For inner nodes of the tree, every entry will steal one bit to mark whether it points to
- * child nodes(sub-block). If so, the corresponding TermStats and TermMetaData are omitted
- * </ul>
- *
- * <a id="Termindex"></a>
- *
- * <h2>Term Index</h2>
- *
- * <p>The .tip file contains an index into the term dictionary, so that it can be accessed randomly.
- * The index is also used to determine when a given term cannot exist on disk (in the .tim file),
- * saving a disk seek.
- *
- * <ul>
- * <li>TermsIndex (.tip) --&gt; Header, FSTIndex<sup>NumFields</sup>
- * &lt;IndexStartFP&gt;<sup>NumFields</sup>, DirOffset, Footer
- * <li>Header --&gt; {@link CodecUtil#writeHeader CodecHeader}
- * <li>DirOffset --&gt; {@link DataOutput#writeLong Uint64}
- * <li>IndexStartFP --&gt; {@link DataOutput#writeVLong VLong}
- * <!-- TODO: better describe FST output here -->
- * <li>FSTIndex --&gt; {@link FST FST&lt;byte[]&gt;}
- * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
- * </ul>
- *
- * <p>Notes:
- *
- * <ul>
- * <li>The .tip file contains a separate FST for each field. The FST maps a term prefix to the
- * on-disk block that holds all terms starting with that prefix. Each field's IndexStartFP
- * points to its FST.
- * <li>DirOffset is a pointer to the start of the IndexStartFPs for all fields
- * <li>It's possible that an on-disk block would contain too many terms (more than the allowed
- * maximum (default: 48)). When this happens, the block is sub-divided into new blocks (called
- * "floor blocks"), and then the output in the FST for the block's prefix encodes the leading
- * byte of each sub-block, and its file pointer.
- * </ul>
- *
- * @see Lucene40BlockTreeTermsReader
- * @lucene.experimental
- */
-public final class Lucene40BlockTreeTermsWriter extends FieldsConsumer {
-
- /**
- * Suggested default value for the {@code minItemsInBlock} parameter to {@link
- * #Lucene40BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)}.
- */
- public static final int DEFAULT_MIN_BLOCK_SIZE = 25;
-
- /**
- * Suggested default value for the {@code maxItemsInBlock} parameter to {@link
- * #Lucene40BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)}.
- */
- public static final int DEFAULT_MAX_BLOCK_SIZE = 48;
-
- // public static boolean DEBUG = false;
- // public static boolean DEBUG2 = false;
-
- // private final static boolean SAVE_DOT_FILES = false;
-
- private final IndexOutput metaOut;
- private final IndexOutput termsOut;
- private final IndexOutput indexOut;
- final int maxDoc;
- final int minItemsInBlock;
- final int maxItemsInBlock;
-
- final PostingsWriterBase postingsWriter;
- final FieldInfos fieldInfos;
-
- private final List<ByteBuffersDataOutput> fields = new ArrayList<>();
-
- /**
- * Create a new writer. The number of items (terms or sub-blocks) per block will aim to be between
- * minItemsPerBlock and maxItemsPerBlock, though in some cases the blocks may be smaller than the
- * min.
- */
- public Lucene40BlockTreeTermsWriter(
- SegmentWriteState state,
- PostingsWriterBase postingsWriter,
- int minItemsInBlock,
- int maxItemsInBlock)
- throws IOException {
- validateSettings(minItemsInBlock, maxItemsInBlock);
-
- this.minItemsInBlock = minItemsInBlock;
- this.maxItemsInBlock = maxItemsInBlock;
-
- this.maxDoc = state.segmentInfo.maxDoc();
- this.fieldInfos = state.fieldInfos;
- this.postingsWriter = postingsWriter;
-
- final String termsName =
- IndexFileNames.segmentFileName(
- state.segmentInfo.name,
- state.segmentSuffix,
- Lucene40BlockTreeTermsReader.TERMS_EXTENSION);
- termsOut = EndiannessReverserUtil.createOutput(state.directory, termsName, state.context);
- boolean success = false;
- IndexOutput metaOut = null, indexOut = null;
- try {
- CodecUtil.writeIndexHeader(
- termsOut,
- Lucene40BlockTreeTermsReader.TERMS_CODEC_NAME,
- Lucene40BlockTreeTermsReader.VERSION_CURRENT,
- state.segmentInfo.getId(),
- state.segmentSuffix);
-
- final String indexName =
- IndexFileNames.segmentFileName(
- state.segmentInfo.name,
- state.segmentSuffix,
- Lucene40BlockTreeTermsReader.TERMS_INDEX_EXTENSION);
- indexOut = EndiannessReverserUtil.createOutput(state.directory, indexName, state.context);
- CodecUtil.writeIndexHeader(
- indexOut,
- Lucene40BlockTreeTermsReader.TERMS_INDEX_CODEC_NAME,
- Lucene40BlockTreeTermsReader.VERSION_CURRENT,
- state.segmentInfo.getId(),
- state.segmentSuffix);
- // segment = state.segmentInfo.name;
-
- final String metaName =
- IndexFileNames.segmentFileName(
- state.segmentInfo.name,
- state.segmentSuffix,
- Lucene40BlockTreeTermsReader.TERMS_META_EXTENSION);
- metaOut = EndiannessReverserUtil.createOutput(state.directory, metaName, state.context);
- CodecUtil.writeIndexHeader(
- metaOut,
- Lucene40BlockTreeTermsReader.TERMS_META_CODEC_NAME,
- Lucene40BlockTreeTermsReader.VERSION_CURRENT,
- state.segmentInfo.getId(),
- state.segmentSuffix);
-
- postingsWriter.init(metaOut, state); // have consumer write its format/header
-
- this.metaOut = metaOut;
- this.indexOut = indexOut;
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(metaOut, termsOut, indexOut);
- }
- }
- }
-
- /** Throws {@code IllegalArgumentException} if any of these settings is invalid. */
- public static void validateSettings(int minItemsInBlock, int maxItemsInBlock) {
- if (minItemsInBlock <= 1) {
- throw new IllegalArgumentException("minItemsInBlock must be >= 2; got " + minItemsInBlock);
- }
- if (minItemsInBlock > maxItemsInBlock) {
- throw new IllegalArgumentException(
- "maxItemsInBlock must be >= minItemsInBlock; got maxItemsInBlock="
- + maxItemsInBlock
- + " minItemsInBlock="
- + minItemsInBlock);
- }
- if (2 * (minItemsInBlock - 1) > maxItemsInBlock) {
- throw new IllegalArgumentException(
- "maxItemsInBlock must be at least 2*(minItemsInBlock-1); got maxItemsInBlock="
- + maxItemsInBlock
- + " minItemsInBlock="
- + minItemsInBlock);
- }
- }
-
- @Override
- public void write(Fields fields, NormsProducer norms) throws IOException {
- // if (DEBUG) System.out.println("\nBTTW.write seg=" + segment);
-
- String lastField = null;
- for (String field : fields) {
- assert lastField == null || lastField.compareTo(field) < 0;
- lastField = field;
-
- // if (DEBUG) System.out.println("\nBTTW.write seg=" + segment + " field=" + field);
- Terms terms = fields.terms(field);
- if (terms == null) {
- continue;
- }
-
- TermsEnum termsEnum = terms.iterator();
- TermsWriter termsWriter = new TermsWriter(fieldInfos.fieldInfo(field));
- while (true) {
- BytesRef term = termsEnum.next();
- // if (DEBUG) System.out.println("BTTW: next term " + term);
-
- if (term == null) {
- break;
- }
-
- // if (DEBUG) System.out.println("write field=" + fieldInfo.name + " term=" +
- // ToStringUtils.bytesRefToString(term));
- termsWriter.write(term, termsEnum, norms);
- }
-
- termsWriter.finish();
-
- // if (DEBUG) System.out.println("\nBTTW.write done seg=" + segment + " field=" + field);
- }
- }
-
- static long encodeOutput(long fp, boolean hasTerms, boolean isFloor) {
- assert fp < (1L << 62);
- return (fp << 2)
- | (hasTerms ? Lucene40BlockTreeTermsReader.OUTPUT_FLAG_HAS_TERMS : 0)
- | (isFloor ? Lucene40BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR : 0);
- }
-
- private static class PendingEntry {
- public final boolean isTerm;
-
- protected PendingEntry(boolean isTerm) {
- this.isTerm = isTerm;
- }
- }
-
- private static final class PendingTerm extends PendingEntry {
- public final byte[] termBytes;
- // stats + metadata
- public final BlockTermState state;
-
- public PendingTerm(BytesRef term, BlockTermState state) {
- super(true);
- this.termBytes = new byte[term.length];
- System.arraycopy(term.bytes, term.offset, termBytes, 0, term.length);
- this.state = state;
- }
-
- @Override
- public String toString() {
- return "TERM: " + ToStringUtils.bytesRefToString(termBytes);
- }
- }
-
- private static final class PendingBlock extends PendingEntry {
- public final BytesRef prefix;
- public final long fp;
- public FST<BytesRef> index;
- public List<FST<BytesRef>> subIndices;
- public final boolean hasTerms;
- public final boolean isFloor;
- public final int floorLeadByte;
-
- public PendingBlock(
- BytesRef prefix,
- long fp,
- boolean hasTerms,
- boolean isFloor,
- int floorLeadByte,
- List<FST<BytesRef>> subIndices) {
- super(false);
- this.prefix = prefix;
- this.fp = fp;
- this.hasTerms = hasTerms;
- this.isFloor = isFloor;
- this.floorLeadByte = floorLeadByte;
- this.subIndices = subIndices;
- }
-
- @Override
- public String toString() {
- return "BLOCK: prefix=" + ToStringUtils.bytesRefToString(prefix);
- }
-
- public void compileIndex(
- List<PendingBlock> blocks,
- ByteBuffersDataOutput scratchBytes,
- IntsRefBuilder scratchIntsRef)
- throws IOException {
-
- assert (isFloor && blocks.size() > 1) || (isFloor == false && blocks.size() == 1)
- : "isFloor=" + isFloor + " blocks=" + blocks;
- assert this == blocks.get(0);
-
- assert scratchBytes.size() == 0;
-
- // TODO: try writing the leading vLong in MSB order
- // (opposite of what Lucene does today), for better
- // outputs sharing in the FST
- scratchBytes.writeVLong(encodeOutput(fp, hasTerms, isFloor));
- if (isFloor) {
- scratchBytes.writeVInt(blocks.size() - 1);
- for (int i = 1; i < blocks.size(); i++) {
- PendingBlock sub = blocks.get(i);
- assert sub.floorLeadByte != -1;
- // if (DEBUG) {
- // System.out.println(" write floorLeadByte=" +
- // Integer.toHexString(sub.floorLeadByte&0xff));
- // }
- scratchBytes.writeByte((byte) sub.floorLeadByte);
- assert sub.fp > fp;
- scratchBytes.writeVLong((sub.fp - fp) << 1 | (sub.hasTerms ? 1 : 0));
- }
- }
-
- final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
- final FSTCompiler<BytesRef> fstCompiler =
- new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs).build();
- // if (DEBUG) {
- // System.out.println(" compile index for prefix=" + prefix);
- // }
- // indexBuilder.DEBUG = false;
- final byte[] bytes = scratchBytes.toArrayCopy();
- assert bytes.length > 0;
- fstCompiler.add(Util.toIntsRef(prefix, scratchIntsRef), new BytesRef(bytes, 0, bytes.length));
- scratchBytes.reset();
-
- // Copy over index for all sub-blocks
- for (PendingBlock block : blocks) {
- if (block.subIndices != null) {
- for (FST<BytesRef> subIndex : block.subIndices) {
- append(fstCompiler, subIndex, scratchIntsRef);
- }
- block.subIndices = null;
- }
- }
-
- index = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
-
- assert subIndices == null;
-
- /*
- Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"));
- Util.toDot(index, w, false, false);
- System.out.println("SAVED to out.dot");
- w.close();
- */
- }
-
- // TODO: maybe we could add bulk-add method to
- // Builder? Takes FST and unions it w/ current
- // FST.
- private void append(
- FSTCompiler<BytesRef> fstCompiler, FST<BytesRef> subIndex, IntsRefBuilder scratchIntsRef)
- throws IOException {
- final BytesRefFSTEnum<BytesRef> subIndexEnum = new BytesRefFSTEnum<>(subIndex);
- BytesRefFSTEnum.InputOutput<BytesRef> indexEnt;
- while ((indexEnt = subIndexEnum.next()) != null) {
- // if (DEBUG) {
- // System.out.println(" add sub=" + indexEnt.input + " " + indexEnt.input + " output="
- // + indexEnt.output);
- // }
- fstCompiler.add(Util.toIntsRef(indexEnt.input, scratchIntsRef), indexEnt.output);
- }
- }
- }
-
- private final ByteBuffersDataOutput scratchBytes = ByteBuffersDataOutput.newResettableInstance();
- private final IntsRefBuilder scratchIntsRef = new IntsRefBuilder();
-
- static final BytesRef EMPTY_BYTES_REF = new BytesRef();
-
- private static class StatsWriter {
-
- private final DataOutput out;
- private final boolean hasFreqs;
- private int singletonCount;
-
- StatsWriter(DataOutput out, boolean hasFreqs) {
- this.out = out;
- this.hasFreqs = hasFreqs;
- }
-
- void add(int df, long ttf) throws IOException {
- // Singletons (DF==1, TTF==1) are run-length encoded
- if (df == 1 && (hasFreqs == false || ttf == 1)) {
- singletonCount++;
- } else {
- finish();
- out.writeVInt(df << 1);
- if (hasFreqs) {
- out.writeVLong(ttf - df);
- }
- }
- }
-
- void finish() throws IOException {
- if (singletonCount > 0) {
- out.writeVInt(((singletonCount - 1) << 1) | 1);
- singletonCount = 0;
- }
- }
- }
-
- class TermsWriter {
- private final FieldInfo fieldInfo;
- private long numTerms;
- final FixedBitSet docsSeen;
- long sumTotalTermFreq;
- long sumDocFreq;
-
- // Records index into pending where the current prefix at that
- // length "started"; for example, if current term starts with 't',
- // startsByPrefix[0] is the index into pending for the first
- // term/sub-block starting with 't'. We use this to figure out when
- // to write a new block:
- private final BytesRefBuilder lastTerm = new BytesRefBuilder();
- private int[] prefixStarts = new int[8];
-
- // Pending stack of terms and blocks. As terms arrive (in sorted order)
- // we append to this stack, and once the top of the stack has enough
- // terms starting with a common prefix, we write a new block with
- // those terms and replace those terms in the stack with a new block:
- private final List<PendingEntry> pending = new ArrayList<>();
-
- // Reused in writeBlocks:
- private final List<PendingBlock> newBlocks = new ArrayList<>();
-
- private PendingTerm firstPendingTerm;
- private PendingTerm lastPendingTerm;
-
- /** Writes the top count entries in pending, using prevTerm to compute the prefix. */
- void writeBlocks(int prefixLength, int count) throws IOException {
-
- assert count > 0;
-
- // if (DEBUG2) {
- // BytesRef br = new BytesRef(lastTerm.bytes());
- // br.length = prefixLength;
- // System.out.println("writeBlocks: seg=" + segment + " prefix=" +
- // ToStringUtils.bytesRefToString(br) + " count=" + count);
- // }
-
- // Root block better write all remaining pending entries:
- assert prefixLength > 0 || count == pending.size();
-
- int lastSuffixLeadLabel = -1;
-
- // True if we saw at least one term in this block (we record if a block
- // only points to sub-blocks in the terms index so we can avoid seeking
- // to it when we are looking for a term):
- boolean hasTerms = false;
- boolean hasSubBlocks = false;
-
- int start = pending.size() - count;
- int end = pending.size();
- int nextBlockStart = start;
- int nextFloorLeadLabel = -1;
-
- for (int i = start; i < end; i++) {
-
- PendingEntry ent = pending.get(i);
-
- int suffixLeadLabel;
-
- if (ent.isTerm) {
- PendingTerm term = (PendingTerm) ent;
- if (term.termBytes.length == prefixLength) {
- // Suffix is 0, i.e. prefix 'foo' and term is
- // 'foo' so the term has empty string suffix
- // in this block
- assert lastSuffixLeadLabel == -1
- : "i=" + i + " lastSuffixLeadLabel=" + lastSuffixLeadLabel;
- suffixLeadLabel = -1;
- } else {
- suffixLeadLabel = term.termBytes[prefixLength] & 0xff;
- }
- } else {
- PendingBlock block = (PendingBlock) ent;
- assert block.prefix.length > prefixLength;
- suffixLeadLabel = block.prefix.bytes[block.prefix.offset + prefixLength] & 0xff;
- }
- // if (DEBUG) System.out.println(" i=" + i + " ent=" + ent + " suffixLeadLabel=" +
- // suffixLeadLabel);
-
- if (suffixLeadLabel != lastSuffixLeadLabel) {
- int itemsInBlock = i - nextBlockStart;
- if (itemsInBlock >= minItemsInBlock && end - nextBlockStart > maxItemsInBlock) {
- // The count is too large for one block, so we must break it into "floor" blocks, where
- // we record
- // the leading label of the suffix of the first term in each floor block, so at search
- // time we can
- // jump to the right floor block. We just use a naive greedy segmenter here: make a new
- // floor
- // block as soon as we have at least minItemsInBlock. This is not always best: it often
- // produces
- // a too-small block as the final block:
- boolean isFloor = itemsInBlock < count;
- newBlocks.add(
- writeBlock(
- prefixLength,
- isFloor,
- nextFloorLeadLabel,
- nextBlockStart,
- i,
- hasTerms,
- hasSubBlocks));
-
- hasTerms = false;
- hasSubBlocks = false;
- nextFloorLeadLabel = suffixLeadLabel;
- nextBlockStart = i;
- }
-
- lastSuffixLeadLabel = suffixLeadLabel;
- }
-
- if (ent.isTerm) {
- hasTerms = true;
- } else {
- hasSubBlocks = true;
- }
- }
-
- // Write last block, if any:
- if (nextBlockStart < end) {
- int itemsInBlock = end - nextBlockStart;
- boolean isFloor = itemsInBlock < count;
- newBlocks.add(
- writeBlock(
- prefixLength,
- isFloor,
- nextFloorLeadLabel,
- nextBlockStart,
- end,
- hasTerms,
- hasSubBlocks));
- }
-
- assert newBlocks.isEmpty() == false;
-
- PendingBlock firstBlock = newBlocks.get(0);
-
- assert firstBlock.isFloor || newBlocks.size() == 1;
-
- firstBlock.compileIndex(newBlocks, scratchBytes, scratchIntsRef);
-
- // Remove slice from the top of the pending stack, that we just wrote:
- pending.subList(pending.size() - count, pending.size()).clear();
-
- // Append new block
- pending.add(firstBlock);
-
- newBlocks.clear();
- }
-
- private boolean allEqual(byte[] b, int startOffset, int endOffset, byte value) {
- Objects.checkFromToIndex(startOffset, endOffset, b.length);
- for (int i = startOffset; i < endOffset; ++i) {
- if (b[i] != value) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Writes the specified slice (start is inclusive, end is exclusive) from pending stack as a new
- * block. If isFloor is true, there were too many (more than maxItemsInBlock) entries sharing
- * the same prefix, and so we broke it into multiple floor blocks where we record the starting
- * label of the suffix of each floor block.
- */
- private PendingBlock writeBlock(
- int prefixLength,
- boolean isFloor,
- int floorLeadLabel,
- int start,
- int end,
- boolean hasTerms,
- boolean hasSubBlocks)
- throws IOException {
-
- assert end > start;
-
- long startFP = termsOut.getFilePointer();
-
- boolean hasFloorLeadLabel = isFloor && floorLeadLabel != -1;
-
- final BytesRef prefix = new BytesRef(prefixLength + (hasFloorLeadLabel ? 1 : 0));
- System.arraycopy(lastTerm.get().bytes, 0, prefix.bytes, 0, prefixLength);
- prefix.length = prefixLength;
-
- // if (DEBUG2) System.out.println(" writeBlock field=" + fieldInfo.name + " prefix=" +
- // ToStringUtils.bytesRefToString(prefix) + " fp=" + startFP + " isFloor=" + isFloor +
- // " isLastInFloor=" + (end == pending.size()) + " floorLeadLabel=" + floorLeadLabel +
- // " start=" + start + " end=" + end + " hasTerms=" + hasTerms + " hasSubBlocks=" +
- // hasSubBlocks);
-
- // Write block header:
- int numEntries = end - start;
- int code = numEntries << 1;
- if (end == pending.size()) {
- // Last block:
- code |= 1;
- }
- termsOut.writeVInt(code);
-
- /*
- if (DEBUG) {
- System.out.println(" writeBlock " + (isFloor ? "(floor) " : "") + "seg=" + segment + " pending.size()=" +
- pending.size() + " prefixLength=" + prefixLength + " indexPrefix=" + ToStringUtils.bytesRefToString(prefix) +
- " entCount=" + (end-start+1) + " startFP=" + startFP + (isFloor ? (" floorLeadLabel=" + Integer.toHexString(floorLeadLabel)) : ""));
- }
- */
-
- // 1st pass: pack term suffix bytes into byte[] blob
- // TODO: cutover to bulk int codec... simple64?
-
- // We optimize the leaf block case (block has only terms), writing a more
- // compact format in this case:
- boolean isLeafBlock = hasSubBlocks == false;
-
- // System.out.println(" isLeaf=" + isLeafBlock);
-
- final List<FST<BytesRef>> subIndices;
-
- boolean absolute = true;
-
- if (isLeafBlock) {
- // Block contains only ordinary terms:
- subIndices = null;
- StatsWriter statsWriter =
- new StatsWriter(this.statsWriter, fieldInfo.getIndexOptions() != IndexOptions.DOCS);
- for (int i = start; i < end; i++) {
- PendingEntry ent = pending.get(i);
- assert ent.isTerm : "i=" + i;
-
- PendingTerm term = (PendingTerm) ent;
-
- assert StringHelper.startsWith(term.termBytes, prefix) : term + " prefix=" + prefix;
- BlockTermState state = term.state;
- final int suffix = term.termBytes.length - prefixLength;
- // if (DEBUG2) {
- // BytesRef suffixBytes = new BytesRef(suffix);
- // System.arraycopy(term.termBytes, prefixLength, suffixBytes.bytes, 0, suffix);
- // suffixBytes.length = suffix;
- // System.out.println(" write term suffix=" +
- // ToStringUtils.bytesRefToString(suffixBytes));
- // }
-
- // For leaf block we write suffix straight
- suffixLengthsWriter.writeVInt(suffix);
- suffixWriter.append(term.termBytes, prefixLength, suffix);
- assert floorLeadLabel == -1 || (term.termBytes[prefixLength] & 0xff) >= floorLeadLabel;
-
- // Write term stats, to separate byte[] blob:
- statsWriter.add(state.docFreq, state.totalTermFreq);
-
- // Write term meta data
- postingsWriter.encodeTerm(metaWriter, fieldInfo, state, absolute);
- absolute = false;
- }
- statsWriter.finish();
- } else {
- // Block has at least one prefix term or a sub block:
- subIndices = new ArrayList<>();
- StatsWriter statsWriter =
- new StatsWriter(this.statsWriter, fieldInfo.getIndexOptions() != IndexOptions.DOCS);
- for (int i = start; i < end; i++) {
- PendingEntry ent = pending.get(i);
- if (ent.isTerm) {
- PendingTerm term = (PendingTerm) ent;
-
- assert StringHelper.startsWith(term.termBytes, prefix) : term + " prefix=" + prefix;
- BlockTermState state = term.state;
- final int suffix = term.termBytes.length - prefixLength;
- // if (DEBUG2) {
- // BytesRef suffixBytes = new BytesRef(suffix);
- // System.arraycopy(term.termBytes, prefixLength, suffixBytes.bytes, 0, suffix);
- // suffixBytes.length = suffix;
- // System.out.println(" write term suffix=" +
- // ToStringUtils.bytesRefToString(suffixBytes));
- // }
-
- // For non-leaf block we borrow 1 bit to record
- // if entry is term or sub-block, and 1 bit to record if
- // it's a prefix term. Terms cannot be larger than ~32 KB
- // so we won't run out of bits:
-
- suffixLengthsWriter.writeVInt(suffix << 1);
- suffixWriter.append(term.termBytes, prefixLength, suffix);
-
- // Write term stats, to separate byte[] blob:
- statsWriter.add(state.docFreq, state.totalTermFreq);
-
- // TODO: now that terms dict "sees" these longs,
- // we can explore better column-stride encodings
- // to encode all long[0]s for this block at
- // once, all long[1]s, etc., e.g. using
- // Simple64. Alternatively, we could interleave
- // stats + meta ... no reason to have them
- // separate anymore:
-
- // Write term meta data
- postingsWriter.encodeTerm(metaWriter, fieldInfo, state, absolute);
- absolute = false;
- } else {
- PendingBlock block = (PendingBlock) ent;
- assert StringHelper.startsWith(block.prefix, prefix);
- final int suffix = block.prefix.length - prefixLength;
- assert StringHelper.startsWith(block.prefix, prefix);
-
- assert suffix > 0;
-
- // For non-leaf block we borrow 1 bit to record
- // if entry is term or sub-block:f
- suffixLengthsWriter.writeVInt((suffix << 1) | 1);
- suffixWriter.append(block.prefix.bytes, prefixLength, suffix);
-
- // if (DEBUG2) {
- // BytesRef suffixBytes = new BytesRef(suffix);
- // System.arraycopy(block.prefix.bytes, prefixLength, suffixBytes.bytes, 0, suffix);
- // suffixBytes.length = suffix;
- // System.out.println(" write sub-block suffix=" +
- // ToStringUtils.bytesRefToString(suffixBytes) +
- // " subFP=" + block.fp + " subCode=" + (startFP-block.fp) + " floor=" + block.isFloor);
- // }
-
- assert floorLeadLabel == -1
- || (block.prefix.bytes[prefixLength] & 0xff) >= floorLeadLabel
- : "floorLeadLabel="
- + floorLeadLabel
- + " suffixLead="
- + (block.prefix.bytes[prefixLength] & 0xff);
- assert block.fp < startFP;
-
- suffixLengthsWriter.writeVLong(startFP - block.fp);
- subIndices.add(block.index);
- }
- }
- statsWriter.finish();
-
- assert subIndices.size() != 0;
- }
-
- // Write suffixes byte[] blob to terms dict output, either uncompressed, compressed with LZ4
- // or with LowercaseAsciiCompression.
- CompressionAlgorithm compressionAlg = CompressionAlgorithm.NO_COMPRESSION;
- // If there are 2 suffix bytes or less per term, then we don't bother compressing as suffix
- // are unlikely what
- // makes the terms dictionary large, and it also tends to be frequently the case for dense IDs
- // like
- // auto-increment IDs, so not compressing in that case helps not hurt ID lookups by too much.
- // We also only start compressing when the prefix length is greater than 2 since blocks whose
- // prefix length is
- // 1 or 2 always all get visited when running a fuzzy query whose max number of edits is 2.
- if (suffixWriter.length() > 2L * numEntries && prefixLength > 2) {
- // LZ4 inserts references whenever it sees duplicate strings of 4 chars or more, so only try
- // it out if the
- // average suffix length is greater than 6.
- if (suffixWriter.length() > 6L * numEntries) {
- LZ4.compress(
- suffixWriter.bytes(), 0, suffixWriter.length(), spareWriter, compressionHashTable);
- if (spareWriter.size() < suffixWriter.length() - (suffixWriter.length() >>> 2)) {
- // LZ4 saved more than 25%, go for it
- compressionAlg = CompressionAlgorithm.LZ4;
- }
- }
- if (compressionAlg == CompressionAlgorithm.NO_COMPRESSION) {
- spareWriter.reset();
- if (spareBytes.length < suffixWriter.length()) {
- spareBytes = new byte[ArrayUtil.oversize(suffixWriter.length(), 1)];
- }
- if (LowercaseAsciiCompression.compress(
- suffixWriter.bytes(), suffixWriter.length(), spareBytes, spareWriter)) {
- compressionAlg = CompressionAlgorithm.LOWERCASE_ASCII;
- }
- }
- }
- long token = ((long) suffixWriter.length()) << 3;
- if (isLeafBlock) {
- token |= 0x04;
- }
- token |= compressionAlg.code;
- termsOut.writeVLong(token);
- if (compressionAlg == CompressionAlgorithm.NO_COMPRESSION) {
- termsOut.writeBytes(suffixWriter.bytes(), suffixWriter.length());
- } else {
- spareWriter.copyTo(termsOut);
- }
- suffixWriter.setLength(0);
- spareWriter.reset();
-
- // Write suffix lengths
- final int numSuffixBytes = Math.toIntExact(suffixLengthsWriter.size());
- spareBytes = ArrayUtil.grow(spareBytes, numSuffixBytes);
- suffixLengthsWriter.copyTo(new ByteArrayDataOutput(spareBytes));
- suffixLengthsWriter.reset();
- if (allEqual(spareBytes, 1, numSuffixBytes, spareBytes[0])) {
- // Structured fields like IDs often have most values of the same length
- termsOut.writeVInt((numSuffixBytes << 1) | 1);
- termsOut.writeByte(spareBytes[0]);
- } else {
- termsOut.writeVInt(numSuffixBytes << 1);
- termsOut.writeBytes(spareBytes, numSuffixBytes);
- }
-
- // Stats
- final int numStatsBytes = Math.toIntExact(statsWriter.size());
- termsOut.writeVInt(numStatsBytes);
- statsWriter.copyTo(termsOut);
- statsWriter.reset();
-
- // Write term meta data byte[] blob
- termsOut.writeVInt((int) metaWriter.size());
- metaWriter.copyTo(termsOut);
- metaWriter.reset();
-
- // if (DEBUG) {
- // System.out.println(" fpEnd=" + out.getFilePointer());
- // }
-
- if (hasFloorLeadLabel) {
- // We already allocated to length+1 above:
- prefix.bytes[prefix.length++] = (byte) floorLeadLabel;
- }
-
- return new PendingBlock(prefix, startFP, hasTerms, isFloor, floorLeadLabel, subIndices);
- }
-
- TermsWriter(FieldInfo fieldInfo) {
- this.fieldInfo = fieldInfo;
- assert fieldInfo.getIndexOptions() != IndexOptions.NONE;
- docsSeen = new FixedBitSet(maxDoc);
- postingsWriter.setField(fieldInfo);
- }
-
- /** Writes one term's worth of postings. */
- public void write(BytesRef text, TermsEnum termsEnum, NormsProducer norms) throws IOException {
- /*
- if (DEBUG) {
- int[] tmp = new int[lastTerm.length];
- System.arraycopy(prefixStarts, 0, tmp, 0, tmp.length);
- System.out.println("BTTW: write term=" + ToStringUtils.bytesRefToString(text) + " prefixStarts=" + Arrays.toString(tmp) +
- " pending.size()=" + pending.size());
- }
- */
-
- BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen, norms);
- if (state != null) {
-
- assert state.docFreq != 0;
- assert fieldInfo.getIndexOptions() == IndexOptions.DOCS
- || state.totalTermFreq >= state.docFreq
- : "postingsWriter=" + postingsWriter;
- pushTerm(text);
-
- PendingTerm term = new PendingTerm(text, state);
- pending.add(term);
- // if (DEBUG) System.out.println(" add pending term = " + text + " pending.size()=" +
- // pending.size());
-
- sumDocFreq += state.docFreq;
- sumTotalTermFreq += state.totalTermFreq;
- numTerms++;
- if (firstPendingTerm == null) {
- firstPendingTerm = term;
- }
- lastPendingTerm = term;
- }
- }
-
- /** Pushes the new term to the top of the stack, and writes new blocks. */
- private void pushTerm(BytesRef text) throws IOException {
- // Find common prefix between last term and current term:
- int prefixLength =
- Arrays.mismatch(
- lastTerm.bytes(),
- 0,
- lastTerm.length(),
- text.bytes,
- text.offset,
- text.offset + text.length);
- if (prefixLength == -1) { // Only happens for the first term, if it is empty
- assert lastTerm.length() == 0;
- prefixLength = 0;
- }
-
- // if (DEBUG) System.out.println(" shared=" + pos + " lastTerm.length=" + lastTerm.length);
-
- // Close the "abandoned" suffix now:
- for (int i = lastTerm.length() - 1; i >= prefixLength; i--) {
-
- // How many items on top of the stack share the current suffix
- // we are closing:
- int prefixTopSize = pending.size() - prefixStarts[i];
- if (prefixTopSize >= minItemsInBlock) {
- // if (DEBUG) System.out.println("pushTerm i=" + i + " prefixTopSize=" + prefixTopSize +
- // " minItemsInBlock=" + minItemsInBlock);
- writeBlocks(i + 1, prefixTopSize);
- prefixStarts[i] -= prefixTopSize - 1;
- }
- }
-
- if (prefixStarts.length < text.length) {
- prefixStarts = ArrayUtil.grow(prefixStarts, text.length);
- }
-
- // Init new tail:
- for (int i = prefixLength; i < text.length; i++) {
- prefixStarts[i] = pending.size();
- }
-
- lastTerm.copyBytes(text);
- }
-
- // Finishes all terms in this field
- public void finish() throws IOException {
- if (numTerms > 0) {
- // if (DEBUG) System.out.println("BTTW: finish prefixStarts=" +
- // Arrays.toString(prefixStarts));
-
- // Add empty term to force closing of all final blocks:
- pushTerm(new BytesRef());
-
- // TODO: if pending.size() is already 1 with a non-zero prefix length
- // we can save writing a "degenerate" root block, but we have to
- // fix all the places that assume the root block's prefix is the empty string:
- pushTerm(new BytesRef());
- writeBlocks(0, pending.size());
-
- // We better have one final "root" block:
- assert pending.size() == 1 && !pending.get(0).isTerm
- : "pending.size()=" + pending.size() + " pending=" + pending;
- final PendingBlock root = (PendingBlock) pending.get(0);
- assert root.prefix.length == 0;
- final BytesRef rootCode = root.index.getEmptyOutput();
- assert rootCode != null;
-
- ByteBuffersDataOutput metaOut = new ByteBuffersDataOutput();
- fields.add(metaOut);
-
- metaOut.writeVInt(fieldInfo.number);
- metaOut.writeVLong(numTerms);
- metaOut.writeVInt(rootCode.length);
- metaOut.writeBytes(rootCode.bytes, rootCode.offset, rootCode.length);
- assert fieldInfo.getIndexOptions() != IndexOptions.NONE;
- if (fieldInfo.getIndexOptions() != IndexOptions.DOCS) {
- metaOut.writeVLong(sumTotalTermFreq);
- }
- metaOut.writeVLong(sumDocFreq);
- metaOut.writeVInt(docsSeen.cardinality());
- writeBytesRef(metaOut, new BytesRef(firstPendingTerm.termBytes));
- writeBytesRef(metaOut, new BytesRef(lastPendingTerm.termBytes));
- metaOut.writeVLong(indexOut.getFilePointer());
- // Write FST to index
- root.index.save(metaOut, indexOut);
- // System.out.println(" write FST " + indexStartFP + " field=" + fieldInfo.name);
-
- /*
- if (DEBUG) {
- final String dotFileName = segment + "_" + fieldInfo.name + ".dot";
- Writer w = new OutputStreamWriter(new FileOutputStream(dotFileName));
- Util.toDot(root.index, w, false, false);
- System.out.println("SAVED to " + dotFileName);
- w.close();
- }
- */
-
- } else {
- assert sumTotalTermFreq == 0
- || fieldInfo.getIndexOptions() == IndexOptions.DOCS && sumTotalTermFreq == -1;
- assert sumDocFreq == 0;
- assert docsSeen.cardinality() == 0;
- }
- }
-
- private final ByteBuffersDataOutput suffixLengthsWriter =
- ByteBuffersDataOutput.newResettableInstance();
- private final BytesRefBuilder suffixWriter = new BytesRefBuilder();
- private final ByteBuffersDataOutput statsWriter = ByteBuffersDataOutput.newResettableInstance();
- private final ByteBuffersDataOutput metaWriter = ByteBuffersDataOutput.newResettableInstance();
- private final ByteBuffersDataOutput spareWriter = ByteBuffersDataOutput.newResettableInstance();
- private byte[] spareBytes = BytesRef.EMPTY_BYTES;
- private final LZ4.HighCompressionHashTable compressionHashTable =
- new LZ4.HighCompressionHashTable();
- }
-
- private boolean closed;
-
- @Override
- public void close() throws IOException {
- if (closed) {
- return;
- }
- closed = true;
-
- boolean success = false;
- try {
- metaOut.writeVInt(fields.size());
- for (ByteBuffersDataOutput fieldMeta : fields) {
- fieldMeta.copyTo(metaOut);
- }
- CodecUtil.writeFooter(indexOut);
- metaOut.writeLong(indexOut.getFilePointer());
- CodecUtil.writeFooter(termsOut);
- metaOut.writeLong(termsOut.getFilePointer());
- CodecUtil.writeFooter(metaOut);
- success = true;
- } finally {
- if (success) {
- IOUtils.close(metaOut, termsOut, indexOut, postingsWriter);
- } else {
- IOUtils.closeWhileHandlingException(metaOut, termsOut, indexOut, postingsWriter);
- }
- }
- }
-
- private static void writeBytesRef(DataOutput out, BytesRef bytes) throws IOException {
- out.writeVInt(bytes.length);
- out.writeBytes(bytes.bytes, bytes.offset, bytes.length);
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene40/blocktree/Lucene40BlockTreeTermsWriterV5.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene40/blocktree/Lucene40BlockTreeTermsWriterV5.java
deleted file mode 100644
index 165f296f282..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene40/blocktree/Lucene40BlockTreeTermsWriterV5.java
+++ /dev/null
@@ -1,1096 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene40.blocktree;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Objects;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.BlockTermState;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.FieldsConsumer;
-import org.apache.lucene.codecs.NormsProducer;
-import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.Fields;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.store.ByteArrayDataOutput;
-import org.apache.lucene.store.ByteBuffersDataOutput;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.FixedBitSet;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.IntsRefBuilder;
-import org.apache.lucene.util.StringHelper;
-import org.apache.lucene.util.compress.LZ4;
-import org.apache.lucene.util.compress.LowercaseAsciiCompression;
-import org.apache.lucene.util.fst.ByteSequenceOutputs;
-import org.apache.lucene.util.fst.BytesRefFSTEnum;
-import org.apache.lucene.util.fst.FST;
-import org.apache.lucene.util.fst.FSTCompiler;
-import org.apache.lucene.util.fst.Util;
-
-/**
- * Writer for {@link Lucene40BlockTreeTermsReader} prior to the refactoring that separated metadata
- * to its own file.
- *
- * @see Lucene40BlockTreeTermsReader#VERSION_META_FILE
- */
-public final class Lucene40BlockTreeTermsWriterV5 extends FieldsConsumer {
-
- /**
- * Suggested default value for the {@code minItemsInBlock} parameter to {@link
- * #Lucene40BlockTreeTermsWriterV5(SegmentWriteState,PostingsWriterBase,int,int)}.
- */
- public static final int DEFAULT_MIN_BLOCK_SIZE = 25;
-
- /**
- * Suggested default value for the {@code maxItemsInBlock} parameter to {@link
- * #Lucene40BlockTreeTermsWriterV5(SegmentWriteState,PostingsWriterBase,int,int)}.
- */
- public static final int DEFAULT_MAX_BLOCK_SIZE = 48;
-
- private static final int VERSION_CURRENT = Lucene40BlockTreeTermsReader.VERSION_META_FILE - 1;
-
- // public static boolean DEBUG = false;
- // public static boolean DEBUG2 = false;
-
- // private final static boolean SAVE_DOT_FILES = false;
-
- private final IndexOutput termsOut;
- private final IndexOutput indexOut;
- final int maxDoc;
- final int minItemsInBlock;
- final int maxItemsInBlock;
-
- final PostingsWriterBase postingsWriter;
- final FieldInfos fieldInfos;
-
- private static class FieldMetaData {
- public final FieldInfo fieldInfo;
- public final BytesRef rootCode;
- public final long numTerms;
- public final long indexStartFP;
- public final long sumTotalTermFreq;
- public final long sumDocFreq;
- public final int docCount;
- public final BytesRef minTerm;
- public final BytesRef maxTerm;
-
- public FieldMetaData(
- FieldInfo fieldInfo,
- BytesRef rootCode,
- long numTerms,
- long indexStartFP,
- long sumTotalTermFreq,
- long sumDocFreq,
- int docCount,
- BytesRef minTerm,
- BytesRef maxTerm) {
- assert numTerms > 0;
- this.fieldInfo = fieldInfo;
- assert rootCode != null : "field=" + fieldInfo.name + " numTerms=" + numTerms;
- this.rootCode = rootCode;
- this.indexStartFP = indexStartFP;
- this.numTerms = numTerms;
- this.sumTotalTermFreq = sumTotalTermFreq;
- this.sumDocFreq = sumDocFreq;
- this.docCount = docCount;
- this.minTerm = minTerm;
- this.maxTerm = maxTerm;
- }
- }
-
- private final List<FieldMetaData> fields = new ArrayList<>();
-
- /**
- * Create a new writer. The number of items (terms or sub-blocks) per block will aim to be between
- * minItemsPerBlock and maxItemsPerBlock, though in some cases the blocks may be smaller than the
- * min.
- */
- public Lucene40BlockTreeTermsWriterV5(
- SegmentWriteState state,
- PostingsWriterBase postingsWriter,
- int minItemsInBlock,
- int maxItemsInBlock)
- throws IOException {
- validateSettings(minItemsInBlock, maxItemsInBlock);
-
- this.minItemsInBlock = minItemsInBlock;
- this.maxItemsInBlock = maxItemsInBlock;
-
- this.maxDoc = state.segmentInfo.maxDoc();
- this.fieldInfos = state.fieldInfos;
- this.postingsWriter = postingsWriter;
-
- final String termsName =
- IndexFileNames.segmentFileName(
- state.segmentInfo.name,
- state.segmentSuffix,
- Lucene40BlockTreeTermsReader.TERMS_EXTENSION);
- termsOut = EndiannessReverserUtil.createOutput(state.directory, termsName, state.context);
- boolean success = false;
- IndexOutput indexOut = null;
- try {
- CodecUtil.writeIndexHeader(
- termsOut,
- Lucene40BlockTreeTermsReader.TERMS_CODEC_NAME,
- VERSION_CURRENT,
- state.segmentInfo.getId(),
- state.segmentSuffix);
-
- final String indexName =
- IndexFileNames.segmentFileName(
- state.segmentInfo.name,
- state.segmentSuffix,
- Lucene40BlockTreeTermsReader.TERMS_INDEX_EXTENSION);
- indexOut = EndiannessReverserUtil.createOutput(state.directory, indexName, state.context);
- CodecUtil.writeIndexHeader(
- indexOut,
- Lucene40BlockTreeTermsReader.TERMS_INDEX_CODEC_NAME,
- VERSION_CURRENT,
- state.segmentInfo.getId(),
- state.segmentSuffix);
- // segment = state.segmentInfo.name;
-
- postingsWriter.init(termsOut, state); // have consumer write its format/header
-
- this.indexOut = indexOut;
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(termsOut, indexOut);
- }
- }
- }
-
- /** Writes the terms file trailer. */
- private void writeTrailer(IndexOutput out, long dirStart) throws IOException {
- out.writeLong(dirStart);
- }
-
- /** Writes the index file trailer. */
- private void writeIndexTrailer(IndexOutput indexOut, long dirStart) throws IOException {
- indexOut.writeLong(dirStart);
- }
-
- /** Throws {@code IllegalArgumentException} if any of these settings is invalid. */
- public static void validateSettings(int minItemsInBlock, int maxItemsInBlock) {
- if (minItemsInBlock <= 1) {
- throw new IllegalArgumentException("minItemsInBlock must be >= 2; got " + minItemsInBlock);
- }
- if (minItemsInBlock > maxItemsInBlock) {
- throw new IllegalArgumentException(
- "maxItemsInBlock must be >= minItemsInBlock; got maxItemsInBlock="
- + maxItemsInBlock
- + " minItemsInBlock="
- + minItemsInBlock);
- }
- if (2 * (minItemsInBlock - 1) > maxItemsInBlock) {
- throw new IllegalArgumentException(
- "maxItemsInBlock must be at least 2*(minItemsInBlock-1); got maxItemsInBlock="
- + maxItemsInBlock
- + " minItemsInBlock="
- + minItemsInBlock);
- }
- }
-
- @Override
- public void write(Fields fields, NormsProducer norms) throws IOException {
- // if (DEBUG) System.out.println("\nBTTW.write seg=" + segment);
-
- String lastField = null;
- for (String field : fields) {
- assert lastField == null || lastField.compareTo(field) < 0;
- lastField = field;
-
- // if (DEBUG) System.out.println("\nBTTW.write seg=" + segment + " field=" + field);
- Terms terms = fields.terms(field);
- if (terms == null) {
- continue;
- }
-
- TermsEnum termsEnum = terms.iterator();
- TermsWriter termsWriter = new TermsWriter(fieldInfos.fieldInfo(field));
- while (true) {
- BytesRef term = termsEnum.next();
- // if (DEBUG) System.out.println("BTTW: next term " + term);
-
- if (term == null) {
- break;
- }
-
- // if (DEBUG) System.out.println("write field=" + fieldInfo.name + " term=" +
- // brToString(term));
- termsWriter.write(term, termsEnum, norms);
- }
-
- termsWriter.finish();
-
- // if (DEBUG) System.out.println("\nBTTW.write done seg=" + segment + " field=" + field);
- }
- }
-
- static long encodeOutput(long fp, boolean hasTerms, boolean isFloor) {
- assert fp < (1L << 62);
- return (fp << 2)
- | (hasTerms ? Lucene40BlockTreeTermsReader.OUTPUT_FLAG_HAS_TERMS : 0)
- | (isFloor ? Lucene40BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR : 0);
- }
-
- private static class PendingEntry {
- public final boolean isTerm;
-
- protected PendingEntry(boolean isTerm) {
- this.isTerm = isTerm;
- }
- }
-
- private static final class PendingTerm extends PendingEntry {
- public final byte[] termBytes;
- // stats + metadata
- public final BlockTermState state;
-
- public PendingTerm(BytesRef term, BlockTermState state) {
- super(true);
- this.termBytes = new byte[term.length];
- System.arraycopy(term.bytes, term.offset, termBytes, 0, term.length);
- this.state = state;
- }
-
- @Override
- public String toString() {
- return "TERM: " + brToString(termBytes);
- }
- }
-
- // for debugging
- @SuppressWarnings("unused")
- static String brToString(BytesRef b) {
- if (b == null) {
- return "(null)";
- } else {
- try {
- return b.utf8ToString() + " " + b;
- } catch (Throwable t) {
- // If BytesRef isn't actually UTF8, or it's eg a
- // prefix of UTF8 that ends mid-unicode-char, we
- // fallback to hex:
- return b.toString();
- }
- }
- }
-
- // for debugging
- @SuppressWarnings("unused")
- static String brToString(byte[] b) {
- return brToString(new BytesRef(b));
- }
-
- private static final class PendingBlock extends PendingEntry {
- public final BytesRef prefix;
- public final long fp;
- public FST<BytesRef> index;
- public List<FST<BytesRef>> subIndices;
- public final boolean hasTerms;
- public final boolean isFloor;
- public final int floorLeadByte;
-
- public PendingBlock(
- BytesRef prefix,
- long fp,
- boolean hasTerms,
- boolean isFloor,
- int floorLeadByte,
- List<FST<BytesRef>> subIndices) {
- super(false);
- this.prefix = prefix;
- this.fp = fp;
- this.hasTerms = hasTerms;
- this.isFloor = isFloor;
- this.floorLeadByte = floorLeadByte;
- this.subIndices = subIndices;
- }
-
- @Override
- public String toString() {
- return "BLOCK: prefix=" + brToString(prefix);
- }
-
- public void compileIndex(
- List<PendingBlock> blocks,
- ByteBuffersDataOutput scratchBytes,
- IntsRefBuilder scratchIntsRef)
- throws IOException {
-
- assert (isFloor && blocks.size() > 1) || (isFloor == false && blocks.size() == 1)
- : "isFloor=" + isFloor + " blocks=" + blocks;
- assert this == blocks.get(0);
-
- assert scratchBytes.size() == 0;
-
- // TODO: try writing the leading vLong in MSB order
- // (opposite of what Lucene does today), for better
- // outputs sharing in the FST
- scratchBytes.writeVLong(encodeOutput(fp, hasTerms, isFloor));
- if (isFloor) {
- scratchBytes.writeVInt(blocks.size() - 1);
- for (int i = 1; i < blocks.size(); i++) {
- PendingBlock sub = blocks.get(i);
- assert sub.floorLeadByte != -1;
- // if (DEBUG) {
- // System.out.println(" write floorLeadByte=" +
- // Integer.toHexString(sub.floorLeadByte&0xff));
- // }
- scratchBytes.writeByte((byte) sub.floorLeadByte);
- assert sub.fp > fp;
- scratchBytes.writeVLong((sub.fp - fp) << 1 | (sub.hasTerms ? 1 : 0));
- }
- }
-
- final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
- final FSTCompiler<BytesRef> fstCompiler =
- new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs).build();
- // if (DEBUG) {
- // System.out.println(" compile index for prefix=" + prefix);
- // }
- // indexBuilder.DEBUG = false;
- final byte[] bytes = scratchBytes.toArrayCopy();
- assert bytes.length > 0;
- fstCompiler.add(Util.toIntsRef(prefix, scratchIntsRef), new BytesRef(bytes, 0, bytes.length));
- scratchBytes.reset();
-
- // Copy over index for all sub-blocks
- for (PendingBlock block : blocks) {
- if (block.subIndices != null) {
- for (FST<BytesRef> subIndex : block.subIndices) {
- append(fstCompiler, subIndex, scratchIntsRef);
- }
- block.subIndices = null;
- }
- }
-
- index = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
-
- assert subIndices == null;
-
- /*
- Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"));
- Util.toDot(index, w, false, false);
- System.out.println("SAVED to out.dot");
- w.close();
- */
- }
-
- // TODO: maybe we could add bulk-add method to
- // Builder? Takes FST and unions it w/ current
- // FST.
- private void append(
- FSTCompiler<BytesRef> fstCompiler, FST<BytesRef> subIndex, IntsRefBuilder scratchIntsRef)
- throws IOException {
- final BytesRefFSTEnum<BytesRef> subIndexEnum = new BytesRefFSTEnum<>(subIndex);
- BytesRefFSTEnum.InputOutput<BytesRef> indexEnt;
- while ((indexEnt = subIndexEnum.next()) != null) {
- // if (DEBUG) {
- // System.out.println(" add sub=" + indexEnt.input + " " + indexEnt.input + " output="
- // + indexEnt.output);
- // }
- fstCompiler.add(Util.toIntsRef(indexEnt.input, scratchIntsRef), indexEnt.output);
- }
- }
- }
-
- private final ByteBuffersDataOutput scratchBytes = ByteBuffersDataOutput.newResettableInstance();
- private final IntsRefBuilder scratchIntsRef = new IntsRefBuilder();
-
- static final BytesRef EMPTY_BYTES_REF = new BytesRef();
-
- private static class StatsWriter {
-
- private final DataOutput out;
- private final boolean hasFreqs;
- private int singletonCount;
-
- StatsWriter(DataOutput out, boolean hasFreqs) {
- this.out = out;
- this.hasFreqs = hasFreqs;
- }
-
- void add(int df, long ttf) throws IOException {
- // Singletons (DF==1, TTF==1) are run-length encoded
- if (df == 1 && (hasFreqs == false || ttf == 1)) {
- singletonCount++;
- } else {
- finish();
- out.writeVInt(df << 1);
- if (hasFreqs) {
- out.writeVLong(ttf - df);
- }
- }
- }
-
- void finish() throws IOException {
- if (singletonCount > 0) {
- out.writeVInt(((singletonCount - 1) << 1) | 1);
- singletonCount = 0;
- }
- }
- }
-
- class TermsWriter {
- private final FieldInfo fieldInfo;
- private long numTerms;
- final FixedBitSet docsSeen;
- long sumTotalTermFreq;
- long sumDocFreq;
- long indexStartFP;
-
- // Records index into pending where the current prefix at that
- // length "started"; for example, if current term starts with 't',
- // startsByPrefix[0] is the index into pending for the first
- // term/sub-block starting with 't'. We use this to figure out when
- // to write a new block:
- private final BytesRefBuilder lastTerm = new BytesRefBuilder();
- private int[] prefixStarts = new int[8];
-
- // Pending stack of terms and blocks. As terms arrive (in sorted order)
- // we append to this stack, and once the top of the stack has enough
- // terms starting with a common prefix, we write a new block with
- // those terms and replace those terms in the stack with a new block:
- private final List<PendingEntry> pending = new ArrayList<>();
-
- // Reused in writeBlocks:
- private final List<PendingBlock> newBlocks = new ArrayList<>();
-
- private PendingTerm firstPendingTerm;
- private PendingTerm lastPendingTerm;
-
- /** Writes the top count entries in pending, using prevTerm to compute the prefix. */
- void writeBlocks(int prefixLength, int count) throws IOException {
-
- assert count > 0;
-
- // if (DEBUG2) {
- // BytesRef br = new BytesRef(lastTerm.bytes());
- // br.length = prefixLength;
- // System.out.println("writeBlocks: seg=" + segment + " prefix=" + brToString(br) + " count="
- // + count);
- // }
-
- // Root block better write all remaining pending entries:
- assert prefixLength > 0 || count == pending.size();
-
- int lastSuffixLeadLabel = -1;
-
- // True if we saw at least one term in this block (we record if a block
- // only points to sub-blocks in the terms index so we can avoid seeking
- // to it when we are looking for a term):
- boolean hasTerms = false;
- boolean hasSubBlocks = false;
-
- int start = pending.size() - count;
- int end = pending.size();
- int nextBlockStart = start;
- int nextFloorLeadLabel = -1;
-
- for (int i = start; i < end; i++) {
-
- PendingEntry ent = pending.get(i);
-
- int suffixLeadLabel;
-
- if (ent.isTerm) {
- PendingTerm term = (PendingTerm) ent;
- if (term.termBytes.length == prefixLength) {
- // Suffix is 0, i.e. prefix 'foo' and term is
- // 'foo' so the term has empty string suffix
- // in this block
- assert lastSuffixLeadLabel == -1
- : "i=" + i + " lastSuffixLeadLabel=" + lastSuffixLeadLabel;
- suffixLeadLabel = -1;
- } else {
- suffixLeadLabel = term.termBytes[prefixLength] & 0xff;
- }
- } else {
- PendingBlock block = (PendingBlock) ent;
- assert block.prefix.length > prefixLength;
- suffixLeadLabel = block.prefix.bytes[block.prefix.offset + prefixLength] & 0xff;
- }
- // if (DEBUG) System.out.println(" i=" + i + " ent=" + ent + " suffixLeadLabel=" +
- // suffixLeadLabel);
-
- if (suffixLeadLabel != lastSuffixLeadLabel) {
- int itemsInBlock = i - nextBlockStart;
- if (itemsInBlock >= minItemsInBlock && end - nextBlockStart > maxItemsInBlock) {
- // The count is too large for one block, so we must break it into "floor" blocks, where
- // we record
- // the leading label of the suffix of the first term in each floor block, so at search
- // time we can
- // jump to the right floor block. We just use a naive greedy segmenter here: make a new
- // floor
- // block as soon as we have at least minItemsInBlock. This is not always best: it often
- // produces
- // a too-small block as the final block:
- boolean isFloor = itemsInBlock < count;
- newBlocks.add(
- writeBlock(
- prefixLength,
- isFloor,
- nextFloorLeadLabel,
- nextBlockStart,
- i,
- hasTerms,
- hasSubBlocks));
-
- hasTerms = false;
- hasSubBlocks = false;
- nextFloorLeadLabel = suffixLeadLabel;
- nextBlockStart = i;
- }
-
- lastSuffixLeadLabel = suffixLeadLabel;
- }
-
- if (ent.isTerm) {
- hasTerms = true;
- } else {
- hasSubBlocks = true;
- }
- }
-
- // Write last block, if any:
- if (nextBlockStart < end) {
- int itemsInBlock = end - nextBlockStart;
- boolean isFloor = itemsInBlock < count;
- newBlocks.add(
- writeBlock(
- prefixLength,
- isFloor,
- nextFloorLeadLabel,
- nextBlockStart,
- end,
- hasTerms,
- hasSubBlocks));
- }
-
- assert newBlocks.isEmpty() == false;
-
- PendingBlock firstBlock = newBlocks.get(0);
-
- assert firstBlock.isFloor || newBlocks.size() == 1;
-
- firstBlock.compileIndex(newBlocks, scratchBytes, scratchIntsRef);
-
- // Remove slice from the top of the pending stack, that we just wrote:
- pending.subList(pending.size() - count, pending.size()).clear();
-
- // Append new block
- pending.add(firstBlock);
-
- newBlocks.clear();
- }
-
- private boolean allEqual(byte[] b, int startOffset, int endOffset, byte value) {
- Objects.checkFromToIndex(startOffset, endOffset, b.length);
- for (int i = startOffset; i < endOffset; ++i) {
- if (b[i] != value) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Writes the specified slice (start is inclusive, end is exclusive) from pending stack as a new
- * block. If isFloor is true, there were too many (more than maxItemsInBlock) entries sharing
- * the same prefix, and so we broke it into multiple floor blocks where we record the starting
- * label of the suffix of each floor block.
- */
- private PendingBlock writeBlock(
- int prefixLength,
- boolean isFloor,
- int floorLeadLabel,
- int start,
- int end,
- boolean hasTerms,
- boolean hasSubBlocks)
- throws IOException {
-
- assert end > start;
-
- long startFP = termsOut.getFilePointer();
-
- boolean hasFloorLeadLabel = isFloor && floorLeadLabel != -1;
-
- final BytesRef prefix = new BytesRef(prefixLength + (hasFloorLeadLabel ? 1 : 0));
- System.arraycopy(lastTerm.get().bytes, 0, prefix.bytes, 0, prefixLength);
- prefix.length = prefixLength;
-
- // if (DEBUG2) System.out.println(" writeBlock field=" + fieldInfo.name + " prefix=" +
- // brToString(prefix) + " fp=" + startFP + " isFloor=" + isFloor + " isLastInFloor=" + (end ==
- // pending.size()) + " floorLeadLabel=" + floorLeadLabel + " start=" + start + " end=" + end +
- // " hasTerms=" + hasTerms + " hasSubBlocks=" + hasSubBlocks);
-
- // Write block header:
- int numEntries = end - start;
- int code = numEntries << 1;
- if (end == pending.size()) {
- // Last block:
- code |= 1;
- }
- termsOut.writeVInt(code);
-
- /*
- if (DEBUG) {
- System.out.println(" writeBlock " + (isFloor ? "(floor) " : "") + "seg=" + segment + " pending.size()=" + pending.size() + " prefixLength=" + prefixLength + " indexPrefix=" + brToString(prefix) + " entCount=" + (end-start+1) + " startFP=" + startFP + (isFloor ? (" floorLeadLabel=" + Integer.toHexString(floorLeadLabel)) : ""));
- }
- */
-
- // 1st pass: pack term suffix bytes into byte[] blob
- // TODO: cutover to bulk int codec... simple64?
-
- // We optimize the leaf block case (block has only terms), writing a more
- // compact format in this case:
- boolean isLeafBlock = hasSubBlocks == false;
-
- // System.out.println(" isLeaf=" + isLeafBlock);
-
- final List<FST<BytesRef>> subIndices;
-
- boolean absolute = true;
-
- if (isLeafBlock) {
- // Block contains only ordinary terms:
- subIndices = null;
- StatsWriter statsWriter =
- new StatsWriter(this.statsWriter, fieldInfo.getIndexOptions() != IndexOptions.DOCS);
- for (int i = start; i < end; i++) {
- PendingEntry ent = pending.get(i);
- assert ent.isTerm : "i=" + i;
-
- PendingTerm term = (PendingTerm) ent;
-
- assert StringHelper.startsWith(term.termBytes, prefix)
- : "term.term=" + new BytesRef(term.termBytes) + " prefix=" + prefix;
- BlockTermState state = term.state;
- final int suffix = term.termBytes.length - prefixLength;
- // if (DEBUG2) {
- // BytesRef suffixBytes = new BytesRef(suffix);
- // System.arraycopy(term.termBytes, prefixLength, suffixBytes.bytes, 0, suffix);
- // suffixBytes.length = suffix;
- // System.out.println(" write term suffix=" + brToString(suffixBytes));
- // }
-
- // For leaf block we write suffix straight
- suffixLengthsWriter.writeVInt(suffix);
- suffixWriter.append(term.termBytes, prefixLength, suffix);
- assert floorLeadLabel == -1 || (term.termBytes[prefixLength] & 0xff) >= floorLeadLabel;
-
- // Write term stats, to separate byte[] blob:
- statsWriter.add(state.docFreq, state.totalTermFreq);
-
- // Write term meta data
- postingsWriter.encodeTerm(metaWriter, fieldInfo, state, absolute);
- absolute = false;
- }
- statsWriter.finish();
- } else {
- // Block has at least one prefix term or a sub block:
- subIndices = new ArrayList<>();
- StatsWriter statsWriter =
- new StatsWriter(this.statsWriter, fieldInfo.getIndexOptions() != IndexOptions.DOCS);
- for (int i = start; i < end; i++) {
- PendingEntry ent = pending.get(i);
- if (ent.isTerm) {
- PendingTerm term = (PendingTerm) ent;
-
- assert StringHelper.startsWith(term.termBytes, prefix)
- : "term.term=" + new BytesRef(term.termBytes) + " prefix=" + prefix;
- BlockTermState state = term.state;
- final int suffix = term.termBytes.length - prefixLength;
- // if (DEBUG2) {
- // BytesRef suffixBytes = new BytesRef(suffix);
- // System.arraycopy(term.termBytes, prefixLength, suffixBytes.bytes, 0, suffix);
- // suffixBytes.length = suffix;
- // System.out.println(" write term suffix=" + brToString(suffixBytes));
- // }
-
- // For non-leaf block we borrow 1 bit to record
- // if entry is term or sub-block, and 1 bit to record if
- // it's a prefix term. Terms cannot be larger than ~32 KB
- // so we won't run out of bits:
-
- suffixLengthsWriter.writeVInt(suffix << 1);
- suffixWriter.append(term.termBytes, prefixLength, suffix);
-
- // Write term stats, to separate byte[] blob:
- statsWriter.add(state.docFreq, state.totalTermFreq);
-
- // TODO: now that terms dict "sees" these longs,
- // we can explore better column-stride encodings
- // to encode all long[0]s for this block at
- // once, all long[1]s, etc., e.g. using
- // Simple64. Alternatively, we could interleave
- // stats + meta ... no reason to have them
- // separate anymore:
-
- // Write term meta data
- postingsWriter.encodeTerm(metaWriter, fieldInfo, state, absolute);
- absolute = false;
- } else {
- PendingBlock block = (PendingBlock) ent;
- assert StringHelper.startsWith(block.prefix, prefix);
- final int suffix = block.prefix.length - prefixLength;
- assert StringHelper.startsWith(block.prefix, prefix);
-
- assert suffix > 0;
-
- // For non-leaf block we borrow 1 bit to record
- // if entry is term or sub-block:f
- suffixLengthsWriter.writeVInt((suffix << 1) | 1);
- suffixWriter.append(block.prefix.bytes, prefixLength, suffix);
-
- // if (DEBUG2) {
- // BytesRef suffixBytes = new BytesRef(suffix);
- // System.arraycopy(block.prefix.bytes, prefixLength, suffixBytes.bytes, 0, suffix);
- // suffixBytes.length = suffix;
- // System.out.println(" write sub-block suffix=" + brToString(suffixBytes) + "
- // subFP=" + block.fp + " subCode=" + (startFP-block.fp) + " floor=" + block.isFloor);
- // }
-
- assert floorLeadLabel == -1
- || (block.prefix.bytes[prefixLength] & 0xff) >= floorLeadLabel
- : "floorLeadLabel="
- + floorLeadLabel
- + " suffixLead="
- + (block.prefix.bytes[prefixLength] & 0xff);
- assert block.fp < startFP;
-
- suffixLengthsWriter.writeVLong(startFP - block.fp);
- subIndices.add(block.index);
- }
- }
- statsWriter.finish();
-
- assert subIndices.size() != 0;
- }
-
- // Write suffixes byte[] blob to terms dict output, either uncompressed, compressed with LZ4
- // or with LowercaseAsciiCompression.
- CompressionAlgorithm compressionAlg = CompressionAlgorithm.NO_COMPRESSION;
- // If there are 2 suffix bytes or less per term, then we don't bother compressing as suffix
- // are unlikely what
- // makes the terms dictionary large, and it also tends to be frequently the case for dense IDs
- // like
- // auto-increment IDs, so not compressing in that case helps not hurt ID lookups by too much.
- // We also only start compressing when the prefix length is greater than 2 since blocks whose
- // prefix length is
- // 1 or 2 always all get visited when running a fuzzy query whose max number of edits is 2.
- if (suffixWriter.length() > 2L * numEntries && prefixLength > 2) {
- // LZ4 inserts references whenever it sees duplicate strings of 4 chars or more, so only try
- // it out if the
- // average suffix length is greater than 6.
- if (suffixWriter.length() > 6L * numEntries) {
- LZ4.compress(
- suffixWriter.bytes(), 0, suffixWriter.length(), spareWriter, compressionHashTable);
- if (spareWriter.size() < suffixWriter.length() - (suffixWriter.length() >>> 2)) {
- // LZ4 saved more than 25%, go for it
- compressionAlg = CompressionAlgorithm.LZ4;
- }
- }
- if (compressionAlg == CompressionAlgorithm.NO_COMPRESSION) {
- spareWriter.reset();
- if (spareBytes.length < suffixWriter.length()) {
- spareBytes = new byte[ArrayUtil.oversize(suffixWriter.length(), 1)];
- }
- if (LowercaseAsciiCompression.compress(
- suffixWriter.bytes(), suffixWriter.length(), spareBytes, spareWriter)) {
- compressionAlg = CompressionAlgorithm.LOWERCASE_ASCII;
- }
- }
- }
- long token = ((long) suffixWriter.length()) << 3;
- if (isLeafBlock) {
- token |= 0x04;
- }
- token |= compressionAlg.code;
- termsOut.writeVLong(token);
- if (compressionAlg == CompressionAlgorithm.NO_COMPRESSION) {
- termsOut.writeBytes(suffixWriter.bytes(), suffixWriter.length());
- } else {
- spareWriter.copyTo(termsOut);
- }
- suffixWriter.setLength(0);
- spareWriter.reset();
-
- // Write suffix lengths
- final int numSuffixBytes = Math.toIntExact(suffixLengthsWriter.size());
- spareBytes = ArrayUtil.grow(spareBytes, numSuffixBytes);
- suffixLengthsWriter.copyTo(new ByteArrayDataOutput(spareBytes));
- suffixLengthsWriter.reset();
- if (allEqual(spareBytes, 1, numSuffixBytes, spareBytes[0])) {
- // Structured fields like IDs often have most values of the same length
- termsOut.writeVInt((numSuffixBytes << 1) | 1);
- termsOut.writeByte(spareBytes[0]);
- } else {
- termsOut.writeVInt(numSuffixBytes << 1);
- termsOut.writeBytes(spareBytes, numSuffixBytes);
- }
-
- // Stats
- final int numStatsBytes = Math.toIntExact(statsWriter.size());
- termsOut.writeVInt(numStatsBytes);
- statsWriter.copyTo(termsOut);
- statsWriter.reset();
-
- // Write term meta data byte[] blob
- termsOut.writeVInt((int) metaWriter.size());
- metaWriter.copyTo(termsOut);
- metaWriter.reset();
-
- // if (DEBUG) {
- // System.out.println(" fpEnd=" + out.getFilePointer());
- // }
-
- if (hasFloorLeadLabel) {
- // We already allocated to length+1 above:
- prefix.bytes[prefix.length++] = (byte) floorLeadLabel;
- }
-
- return new PendingBlock(prefix, startFP, hasTerms, isFloor, floorLeadLabel, subIndices);
- }
-
- TermsWriter(FieldInfo fieldInfo) {
- this.fieldInfo = fieldInfo;
- assert fieldInfo.getIndexOptions() != IndexOptions.NONE;
- docsSeen = new FixedBitSet(maxDoc);
- postingsWriter.setField(fieldInfo);
- }
-
- /** Writes one term's worth of postings. */
- public void write(BytesRef text, TermsEnum termsEnum, NormsProducer norms) throws IOException {
- /*
- if (DEBUG) {
- int[] tmp = new int[lastTerm.length];
- System.arraycopy(prefixStarts, 0, tmp, 0, tmp.length);
- System.out.println("BTTW: write term=" + brToString(text) + " prefixStarts=" + Arrays.toString(tmp) + " pending.size()=" + pending.size());
- }
- */
-
- BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen, norms);
- if (state != null) {
-
- assert state.docFreq != 0;
- assert fieldInfo.getIndexOptions() == IndexOptions.DOCS
- || state.totalTermFreq >= state.docFreq
- : "postingsWriter=" + postingsWriter;
- pushTerm(text);
-
- PendingTerm term = new PendingTerm(text, state);
- pending.add(term);
- // if (DEBUG) System.out.println(" add pending term = " + text + " pending.size()=" +
- // pending.size());
-
- sumDocFreq += state.docFreq;
- sumTotalTermFreq += state.totalTermFreq;
- numTerms++;
- if (firstPendingTerm == null) {
- firstPendingTerm = term;
- }
- lastPendingTerm = term;
- }
- }
-
- /** Pushes the new term to the top of the stack, and writes new blocks. */
- private void pushTerm(BytesRef text) throws IOException {
- // Find common prefix between last term and current term:
- int prefixLength =
- Arrays.mismatch(
- lastTerm.bytes(),
- 0,
- lastTerm.length(),
- text.bytes,
- text.offset,
- text.offset + text.length);
- if (prefixLength == -1) { // Only happens for the first term, if it is empty
- assert lastTerm.length() == 0;
- prefixLength = 0;
- }
-
- // if (DEBUG) System.out.println(" shared=" + pos + " lastTerm.length=" + lastTerm.length);
-
- // Close the "abandoned" suffix now:
- for (int i = lastTerm.length() - 1; i >= prefixLength; i--) {
-
- // How many items on top of the stack share the current suffix
- // we are closing:
- int prefixTopSize = pending.size() - prefixStarts[i];
- if (prefixTopSize >= minItemsInBlock) {
- // if (DEBUG) System.out.println("pushTerm i=" + i + " prefixTopSize=" + prefixTopSize + "
- // minItemsInBlock=" + minItemsInBlock);
- writeBlocks(i + 1, prefixTopSize);
- prefixStarts[i] -= prefixTopSize - 1;
- }
- }
-
- if (prefixStarts.length < text.length) {
- prefixStarts = ArrayUtil.grow(prefixStarts, text.length);
- }
-
- // Init new tail:
- for (int i = prefixLength; i < text.length; i++) {
- prefixStarts[i] = pending.size();
- }
-
- lastTerm.copyBytes(text);
- }
-
- // Finishes all terms in this field
- public void finish() throws IOException {
- if (numTerms > 0) {
- // if (DEBUG) System.out.println("BTTW: finish prefixStarts=" +
- // Arrays.toString(prefixStarts));
-
- // Add empty term to force closing of all final blocks:
- pushTerm(new BytesRef());
-
- // TODO: if pending.size() is already 1 with a non-zero prefix length
- // we can save writing a "degenerate" root block, but we have to
- // fix all the places that assume the root block's prefix is the empty string:
- pushTerm(new BytesRef());
- writeBlocks(0, pending.size());
-
- // We better have one final "root" block:
- assert pending.size() == 1 && !pending.get(0).isTerm
- : "pending.size()=" + pending.size() + " pending=" + pending;
- final PendingBlock root = (PendingBlock) pending.get(0);
- assert root.prefix.length == 0;
- assert root.index.getEmptyOutput() != null;
-
- // Write FST to index
- indexStartFP = indexOut.getFilePointer();
- root.index.save(indexOut, indexOut);
- // System.out.println(" write FST " + indexStartFP + " field=" + fieldInfo.name);
-
- /*
- if (DEBUG) {
- final String dotFileName = segment + "_" + fieldInfo.name + ".dot";
- Writer w = new OutputStreamWriter(new FileOutputStream(dotFileName));
- Util.toDot(root.index, w, false, false);
- System.out.println("SAVED to " + dotFileName);
- w.close();
- }
- */
- assert firstPendingTerm != null;
- BytesRef minTerm = new BytesRef(firstPendingTerm.termBytes);
-
- assert lastPendingTerm != null;
- BytesRef maxTerm = new BytesRef(lastPendingTerm.termBytes);
-
- fields.add(
- new FieldMetaData(
- fieldInfo,
- ((PendingBlock) pending.get(0)).index.getEmptyOutput(),
- numTerms,
- indexStartFP,
- sumTotalTermFreq,
- sumDocFreq,
- docsSeen.cardinality(),
- minTerm,
- maxTerm));
- } else {
- assert sumTotalTermFreq == 0
- || fieldInfo.getIndexOptions() == IndexOptions.DOCS && sumTotalTermFreq == -1;
- assert sumDocFreq == 0;
- assert docsSeen.cardinality() == 0;
- }
- }
-
- private final ByteBuffersDataOutput suffixLengthsWriter =
- ByteBuffersDataOutput.newResettableInstance();
- private final BytesRefBuilder suffixWriter = new BytesRefBuilder();
- private final ByteBuffersDataOutput statsWriter = ByteBuffersDataOutput.newResettableInstance();
- private final ByteBuffersDataOutput metaWriter = ByteBuffersDataOutput.newResettableInstance();
- private final ByteBuffersDataOutput spareWriter = ByteBuffersDataOutput.newResettableInstance();
- private byte[] spareBytes = BytesRef.EMPTY_BYTES;
- private final LZ4.HighCompressionHashTable compressionHashTable =
- new LZ4.HighCompressionHashTable();
- }
-
- private boolean closed;
-
- @Override
- public void close() throws IOException {
- if (closed) {
- return;
- }
- closed = true;
-
- boolean success = false;
- try {
-
- final long dirStart = termsOut.getFilePointer();
- final long indexDirStart = indexOut.getFilePointer();
-
- termsOut.writeVInt(fields.size());
-
- for (FieldMetaData field : fields) {
- // System.out.println(" field " + field.fieldInfo.name + " " + field.numTerms + " terms");
- termsOut.writeVInt(field.fieldInfo.number);
- assert field.numTerms > 0;
- termsOut.writeVLong(field.numTerms);
- termsOut.writeVInt(field.rootCode.length);
- termsOut.writeBytes(field.rootCode.bytes, field.rootCode.offset, field.rootCode.length);
- assert field.fieldInfo.getIndexOptions() != IndexOptions.NONE;
- if (field.fieldInfo.getIndexOptions() != IndexOptions.DOCS) {
- termsOut.writeVLong(field.sumTotalTermFreq);
- }
- termsOut.writeVLong(field.sumDocFreq);
- termsOut.writeVInt(field.docCount);
- indexOut.writeVLong(field.indexStartFP);
- writeBytesRef(termsOut, field.minTerm);
- writeBytesRef(termsOut, field.maxTerm);
- }
- writeTrailer(termsOut, dirStart);
- CodecUtil.writeFooter(termsOut);
- writeIndexTrailer(indexOut, indexDirStart);
- CodecUtil.writeFooter(indexOut);
- success = true;
- } finally {
- if (success) {
- IOUtils.close(termsOut, indexOut, postingsWriter);
- } else {
- IOUtils.closeWhileHandlingException(termsOut, indexOut, postingsWriter);
- }
- }
- }
-
- private static void writeBytesRef(IndexOutput out, BytesRef bytes) throws IOException {
- out.writeVInt(bytes.length);
- out.writeBytes(bytes.bytes, bytes.offset, bytes.length);
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50PostingsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50PostingsWriter.java
deleted file mode 100644
index ad44548942b..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50PostingsWriter.java
+++ /dev/null
@@ -1,529 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import static org.apache.lucene.backward_codecs.lucene50.ForUtil.MAX_DATA_SIZE;
-import static org.apache.lucene.backward_codecs.lucene50.ForUtil.MAX_ENCODED_SIZE;
-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.BLOCK_SIZE;
-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.DOC_CODEC;
-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.MAX_SKIP_LEVELS;
-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.PAY_CODEC;
-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.POS_CODEC;
-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.TERMS_CODEC;
-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.VERSION_CURRENT;
-
-import java.io.IOException;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.IntBlockTermState;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.BlockTermState;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
-import org.apache.lucene.codecs.PushPostingsWriterBase;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.packed.PackedInts;
-
-/**
- * Concrete class that writes docId(maybe frq,pos,offset,payloads) list with postings format.
- *
- * <p>Postings list for each term will be stored separately.
- *
- * @see Lucene50SkipWriter for details about skipping setting and postings layout.
- * @lucene.experimental
- */
-public final class Lucene50PostingsWriter extends PushPostingsWriterBase {
-
- IndexOutput docOut;
- IndexOutput posOut;
- IndexOutput payOut;
-
- static final IntBlockTermState emptyState = new IntBlockTermState();
- IntBlockTermState lastState;
-
- // Holds starting file pointers for current term:
- private long docStartFP;
- private long posStartFP;
- private long payStartFP;
-
- final int[] docDeltaBuffer;
- final int[] freqBuffer;
- private int docBufferUpto;
-
- final int[] posDeltaBuffer;
- final int[] payloadLengthBuffer;
- final int[] offsetStartDeltaBuffer;
- final int[] offsetLengthBuffer;
- private int posBufferUpto;
-
- private byte[] payloadBytes;
- private int payloadByteUpto;
-
- private int lastBlockDocID;
- private long lastBlockPosFP;
- private long lastBlockPayFP;
- private int lastBlockPosBufferUpto;
- private int lastBlockPayloadByteUpto;
-
- private int lastDocID;
- private int lastPosition;
- private int lastStartOffset;
- private int docCount;
-
- final byte[] encoded;
-
- private final ForUtil forUtil;
- private final Lucene50SkipWriter skipWriter;
-
- private boolean fieldHasNorms;
- private NumericDocValues norms;
- private final CompetitiveImpactAccumulator competitiveFreqNormAccumulator =
- new CompetitiveImpactAccumulator();
-
- /** Creates a postings writer */
- public Lucene50PostingsWriter(SegmentWriteState state) throws IOException {
- final float acceptableOverheadRatio = PackedInts.COMPACT;
-
- String docFileName =
- IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, Lucene50PostingsFormat.DOC_EXTENSION);
- docOut = EndiannessReverserUtil.createOutput(state.directory, docFileName, state.context);
- IndexOutput posOut = null;
- IndexOutput payOut = null;
- boolean success = false;
- try {
- CodecUtil.writeIndexHeader(
- docOut, DOC_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
- forUtil = new ForUtil(acceptableOverheadRatio, docOut);
- if (state.fieldInfos.hasProx()) {
- posDeltaBuffer = new int[MAX_DATA_SIZE];
- String posFileName =
- IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, Lucene50PostingsFormat.POS_EXTENSION);
- posOut = EndiannessReverserUtil.createOutput(state.directory, posFileName, state.context);
- CodecUtil.writeIndexHeader(
- posOut, POS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
-
- if (state.fieldInfos.hasPayloads()) {
- payloadBytes = new byte[128];
- payloadLengthBuffer = new int[MAX_DATA_SIZE];
- } else {
- payloadBytes = null;
- payloadLengthBuffer = null;
- }
-
- if (state.fieldInfos.hasOffsets()) {
- offsetStartDeltaBuffer = new int[MAX_DATA_SIZE];
- offsetLengthBuffer = new int[MAX_DATA_SIZE];
- } else {
- offsetStartDeltaBuffer = null;
- offsetLengthBuffer = null;
- }
-
- if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {
- String payFileName =
- IndexFileNames.segmentFileName(
- state.segmentInfo.name,
- state.segmentSuffix,
- Lucene50PostingsFormat.PAY_EXTENSION);
- payOut = EndiannessReverserUtil.createOutput(state.directory, payFileName, state.context);
- CodecUtil.writeIndexHeader(
- payOut, PAY_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
- }
- } else {
- posDeltaBuffer = null;
- payloadLengthBuffer = null;
- offsetStartDeltaBuffer = null;
- offsetLengthBuffer = null;
- payloadBytes = null;
- }
- this.payOut = payOut;
- this.posOut = posOut;
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(docOut, posOut, payOut);
- }
- }
-
- docDeltaBuffer = new int[MAX_DATA_SIZE];
- freqBuffer = new int[MAX_DATA_SIZE];
-
- // TODO: should we try skipping every 2/4 blocks...?
- skipWriter =
- new Lucene50SkipWriter(
- MAX_SKIP_LEVELS, BLOCK_SIZE, state.segmentInfo.maxDoc(), docOut, posOut, payOut);
-
- encoded = new byte[MAX_ENCODED_SIZE];
- }
-
- @Override
- public IntBlockTermState newTermState() {
- return new IntBlockTermState();
- }
-
- @Override
- public void init(IndexOutput termsOut, SegmentWriteState state) throws IOException {
- CodecUtil.writeIndexHeader(
- termsOut, TERMS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
- termsOut.writeVInt(BLOCK_SIZE);
- }
-
- @Override
- public void setField(FieldInfo fieldInfo) {
- super.setField(fieldInfo);
- skipWriter.setField(writePositions, writeOffsets, writePayloads);
- lastState = emptyState;
- fieldHasNorms = fieldInfo.hasNorms();
- }
-
- @Override
- public void startTerm(NumericDocValues norms) {
- docStartFP = docOut.getFilePointer();
- if (writePositions) {
- posStartFP = posOut.getFilePointer();
- if (writePayloads || writeOffsets) {
- payStartFP = payOut.getFilePointer();
- }
- }
- lastDocID = 0;
- lastBlockDocID = -1;
- skipWriter.resetSkip();
- this.norms = norms;
- competitiveFreqNormAccumulator.clear();
- }
-
- @Override
- public void startDoc(int docID, int termDocFreq) throws IOException {
- // Have collected a block of docs, and get a new doc.
- // Should write skip data as well as postings list for
- // current block.
- if (lastBlockDocID != -1 && docBufferUpto == 0) {
- skipWriter.bufferSkip(
- lastBlockDocID,
- competitiveFreqNormAccumulator,
- docCount,
- lastBlockPosFP,
- lastBlockPayFP,
- lastBlockPosBufferUpto,
- lastBlockPayloadByteUpto);
- competitiveFreqNormAccumulator.clear();
- }
-
- final int docDelta = docID - lastDocID;
-
- if (docID < 0 || (docCount > 0 && docDelta <= 0)) {
- throw new CorruptIndexException(
- "docs out of order (" + docID + " <= " + lastDocID + " )", docOut);
- }
-
- docDeltaBuffer[docBufferUpto] = docDelta;
- if (writeFreqs) {
- freqBuffer[docBufferUpto] = termDocFreq;
- }
-
- docBufferUpto++;
- docCount++;
-
- if (docBufferUpto == BLOCK_SIZE) {
- forUtil.writeBlock(docDeltaBuffer, encoded, docOut);
- if (writeFreqs) {
- forUtil.writeBlock(freqBuffer, encoded, docOut);
- }
- // NOTE: don't set docBufferUpto back to 0 here;
- // finishDoc will do so (because it needs to see that
- // the block was filled so it can save skip data)
- }
-
- lastDocID = docID;
- lastPosition = 0;
- lastStartOffset = 0;
-
- long norm;
- if (fieldHasNorms) {
- boolean found = norms.advanceExact(docID);
- if (found == false) {
- // This can happen if indexing hits a problem after adding a doc to the
- // postings but before buffering the norm. Such documents are written
- // deleted and will go away on the first merge.
- norm = 1L;
- } else {
- norm = norms.longValue();
- assert norm != 0 : docID;
- }
- } else {
- norm = 1L;
- }
-
- competitiveFreqNormAccumulator.add(writeFreqs ? termDocFreq : 1, norm);
- }
-
- @Override
- public void addPosition(int position, BytesRef payload, int startOffset, int endOffset)
- throws IOException {
- if (position > IndexWriter.MAX_POSITION) {
- throw new CorruptIndexException(
- "position="
- + position
- + " is too large (> IndexWriter.MAX_POSITION="
- + IndexWriter.MAX_POSITION
- + ")",
- docOut);
- }
- if (position < 0) {
- throw new CorruptIndexException("position=" + position + " is < 0", docOut);
- }
- posDeltaBuffer[posBufferUpto] = position - lastPosition;
- if (writePayloads) {
- if (payload == null || payload.length == 0) {
- // no payload
- payloadLengthBuffer[posBufferUpto] = 0;
- } else {
- payloadLengthBuffer[posBufferUpto] = payload.length;
- if (payloadByteUpto + payload.length > payloadBytes.length) {
- payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payload.length);
- }
- System.arraycopy(
- payload.bytes, payload.offset, payloadBytes, payloadByteUpto, payload.length);
- payloadByteUpto += payload.length;
- }
- }
-
- if (writeOffsets) {
- assert startOffset >= lastStartOffset;
- assert endOffset >= startOffset;
- offsetStartDeltaBuffer[posBufferUpto] = startOffset - lastStartOffset;
- offsetLengthBuffer[posBufferUpto] = endOffset - startOffset;
- lastStartOffset = startOffset;
- }
-
- posBufferUpto++;
- lastPosition = position;
- if (posBufferUpto == BLOCK_SIZE) {
- forUtil.writeBlock(posDeltaBuffer, encoded, posOut);
-
- if (writePayloads) {
- forUtil.writeBlock(payloadLengthBuffer, encoded, payOut);
- payOut.writeVInt(payloadByteUpto);
- payOut.writeBytes(payloadBytes, 0, payloadByteUpto);
- payloadByteUpto = 0;
- }
- if (writeOffsets) {
- forUtil.writeBlock(offsetStartDeltaBuffer, encoded, payOut);
- forUtil.writeBlock(offsetLengthBuffer, encoded, payOut);
- }
- posBufferUpto = 0;
- }
- }
-
- @Override
- public void finishDoc() throws IOException {
- // Since we don't know df for current term, we had to buffer
- // those skip data for each block, and when a new doc comes,
- // write them to skip file.
- if (docBufferUpto == BLOCK_SIZE) {
- lastBlockDocID = lastDocID;
- if (posOut != null) {
- if (payOut != null) {
- lastBlockPayFP = payOut.getFilePointer();
- }
- lastBlockPosFP = posOut.getFilePointer();
- lastBlockPosBufferUpto = posBufferUpto;
- lastBlockPayloadByteUpto = payloadByteUpto;
- }
- docBufferUpto = 0;
- }
- }
-
- /** Called when we are done adding docs to this term */
- @Override
- public void finishTerm(BlockTermState _state) throws IOException {
- IntBlockTermState state = (IntBlockTermState) _state;
- assert state.docFreq > 0;
-
- // TODO: wasteful we are counting this (counting # docs
- // for this term) in two places?
- assert state.docFreq == docCount : state.docFreq + " vs " + docCount;
-
- // docFreq == 1, don't write the single docid/freq to a separate file along with a pointer to
- // it.
- final int singletonDocID;
- if (state.docFreq == 1) {
- // pulse the singleton docid into the term dictionary, freq is implicitly totalTermFreq
- singletonDocID = docDeltaBuffer[0];
- } else {
- singletonDocID = -1;
- // vInt encode the remaining doc deltas and freqs:
- for (int i = 0; i < docBufferUpto; i++) {
- final int docDelta = docDeltaBuffer[i];
- final int freq = freqBuffer[i];
- if (!writeFreqs) {
- docOut.writeVInt(docDelta);
- } else if (freqBuffer[i] == 1) {
- docOut.writeVInt((docDelta << 1) | 1);
- } else {
- docOut.writeVInt(docDelta << 1);
- docOut.writeVInt(freq);
- }
- }
- }
-
- final long lastPosBlockOffset;
-
- if (writePositions) {
- // totalTermFreq is just total number of positions(or payloads, or offsets)
- // associated with current term.
- assert state.totalTermFreq != -1;
- if (state.totalTermFreq > BLOCK_SIZE) {
- // record file offset for last pos in last block
- lastPosBlockOffset = posOut.getFilePointer() - posStartFP;
- } else {
- lastPosBlockOffset = -1;
- }
- if (posBufferUpto > 0) {
- // TODO: should we send offsets/payloads to
- // .pay...? seems wasteful (have to store extra
- // vLong for low (< BLOCK_SIZE) DF terms = vast vast
- // majority)
-
- // vInt encode the remaining positions/payloads/offsets:
- int lastPayloadLength = -1; // force first payload length to be written
- int lastOffsetLength = -1; // force first offset length to be written
- int payloadBytesReadUpto = 0;
- for (int i = 0; i < posBufferUpto; i++) {
- final int posDelta = posDeltaBuffer[i];
- if (writePayloads) {
- final int payloadLength = payloadLengthBuffer[i];
- if (payloadLength != lastPayloadLength) {
- lastPayloadLength = payloadLength;
- posOut.writeVInt((posDelta << 1) | 1);
- posOut.writeVInt(payloadLength);
- } else {
- posOut.writeVInt(posDelta << 1);
- }
-
- if (payloadLength != 0) {
- posOut.writeBytes(payloadBytes, payloadBytesReadUpto, payloadLength);
- payloadBytesReadUpto += payloadLength;
- }
- } else {
- posOut.writeVInt(posDelta);
- }
-
- if (writeOffsets) {
- int delta = offsetStartDeltaBuffer[i];
- int length = offsetLengthBuffer[i];
- if (length == lastOffsetLength) {
- posOut.writeVInt(delta << 1);
- } else {
- posOut.writeVInt(delta << 1 | 1);
- posOut.writeVInt(length);
- lastOffsetLength = length;
- }
- }
- }
-
- if (writePayloads) {
- assert payloadBytesReadUpto == payloadByteUpto;
- payloadByteUpto = 0;
- }
- }
- } else {
- lastPosBlockOffset = -1;
- }
-
- long skipOffset;
- if (docCount > BLOCK_SIZE) {
- skipOffset = skipWriter.writeSkip(docOut) - docStartFP;
- } else {
- skipOffset = -1;
- }
-
- state.docStartFP = docStartFP;
- state.posStartFP = posStartFP;
- state.payStartFP = payStartFP;
- state.singletonDocID = singletonDocID;
- state.skipOffset = skipOffset;
- state.lastPosBlockOffset = lastPosBlockOffset;
- docBufferUpto = 0;
- posBufferUpto = 0;
- lastDocID = 0;
- docCount = 0;
- }
-
- @Override
- public void encodeTerm(
- DataOutput out, FieldInfo fieldInfo, BlockTermState _state, boolean absolute)
- throws IOException {
- IntBlockTermState state = (IntBlockTermState) _state;
- if (absolute) {
- lastState = emptyState;
- }
- out.writeVLong(state.docStartFP - lastState.docStartFP);
- if (writePositions) {
- out.writeVLong(state.posStartFP - lastState.posStartFP);
- if (writePayloads || writeOffsets) {
- out.writeVLong(state.payStartFP - lastState.payStartFP);
- }
- }
- if (state.singletonDocID != -1) {
- out.writeVInt(state.singletonDocID);
- }
- if (writePositions) {
- if (state.lastPosBlockOffset != -1) {
- out.writeVLong(state.lastPosBlockOffset);
- }
- }
- if (state.skipOffset != -1) {
- out.writeVLong(state.skipOffset);
- }
- lastState = state;
- }
-
- @Override
- public void close() throws IOException {
- // TODO: add a finish() at least to PushBase? DV too...?
- boolean success = false;
- try {
- if (docOut != null) {
- CodecUtil.writeFooter(docOut);
- }
- if (posOut != null) {
- CodecUtil.writeFooter(posOut);
- }
- if (payOut != null) {
- CodecUtil.writeFooter(payOut);
- }
- success = true;
- } finally {
- if (success) {
- IOUtils.close(docOut, posOut, payOut);
- } else {
- IOUtils.closeWhileHandlingException(docOut, posOut, payOut);
- }
- docOut = posOut = payOut = null;
- }
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWCompoundFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWCompoundFormat.java
deleted file mode 100644
index cfd269d9ddd..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWCompoundFormat.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import java.io.IOException;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.CompoundDirectory;
-import org.apache.lucene.codecs.CompoundFormat;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexOutput;
-
-/**
- * Lucene 5.0 compound file format
- *
- * <p>Files:
- *
- * <ul>
- * <li><code>.cfs</code>: An optional "virtual" file consisting of all the other index files for
- * systems that frequently run out of file handles.
- * <li><code>.cfe</code>: The "virtual" compound file's entry table holding all entries in the
- * corresponding .cfs file.
- * </ul>
- *
- * <p>Description:
- *
- * <ul>
- * <li>Compound (.cfs) --&gt; Header, FileData <sup>FileCount</sup>, Footer
- * <li>Compound Entry Table (.cfe) --&gt; Header, FileCount, &lt;FileName, DataOffset,
- * DataLength&gt; <sup>FileCount</sup>
- * <li>Header --&gt; {@link CodecUtil#writeIndexHeader IndexHeader}
- * <li>FileCount --&gt; {@link DataOutput#writeVInt VInt}
- * <li>DataOffset,DataLength,Checksum --&gt; {@link DataOutput#writeLong UInt64}
- * <li>FileName --&gt; {@link DataOutput#writeString String}
- * <li>FileData --&gt; raw file data
- * <li>Footer --&gt; {@link CodecUtil#writeFooter CodecFooter}
- * </ul>
- *
- * <p>Notes:
- *
- * <ul>
- * <li>FileCount indicates how many files are contained in this compound file. The entry table
- * that follows has that many entries.
- * <li>Each directory entry contains a long pointer to the start of this file's data section, the
- * files length, and a String with that file's name.
- * </ul>
- */
-public final class Lucene50RWCompoundFormat extends CompoundFormat {
-
- /** Extension of compound file */
- static final String DATA_EXTENSION = "cfs";
-
- /** Extension of compound file entries */
- static final String ENTRIES_EXTENSION = "cfe";
-
- static final String DATA_CODEC = "Lucene50CompoundData";
- static final String ENTRY_CODEC = "Lucene50CompoundEntries";
- static final int VERSION_START = 0;
- static final int VERSION_CURRENT = VERSION_START;
-
- /** Sole constructor. */
- public Lucene50RWCompoundFormat() {}
-
- @Override
- public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si) throws IOException {
- return new Lucene50CompoundReader(dir, si);
- }
-
- @Override
- public void write(Directory dir, SegmentInfo si, IOContext context) throws IOException {
- String dataFile = IndexFileNames.segmentFileName(si.name, "", DATA_EXTENSION);
- String entriesFile = IndexFileNames.segmentFileName(si.name, "", ENTRIES_EXTENSION);
-
- try (IndexOutput data = EndiannessReverserUtil.createOutput(dir, dataFile, context);
- IndexOutput entries = EndiannessReverserUtil.createOutput(dir, entriesFile, context)) {
- CodecUtil.writeIndexHeader(data, DATA_CODEC, VERSION_CURRENT, si.getId(), "");
- CodecUtil.writeIndexHeader(entries, ENTRY_CODEC, VERSION_CURRENT, si.getId(), "");
-
- writeCompoundFile(entries, data, dir, si);
-
- CodecUtil.writeFooter(data);
- CodecUtil.writeFooter(entries);
- }
- }
-
- private void writeCompoundFile(
- IndexOutput entries, IndexOutput data, Directory dir, SegmentInfo si) throws IOException {
- // write number of files
- entries.writeVInt(si.files().size());
- for (String file : si.files()) {
- // write bytes for file
- long startOffset = data.getFilePointer();
- try (ChecksumIndexInput in =
- EndiannessReverserUtil.openChecksumInput(dir, file, IOContext.READONCE)) {
-
- // just copies the index header, verifying that its id matches what we expect
- CodecUtil.verifyAndCopyIndexHeader(in, data, si.getId());
-
- // copy all bytes except the footer
- long numBytesToCopy = in.length() - CodecUtil.footerLength() - in.getFilePointer();
- data.copyBytes(in, numBytesToCopy);
-
- // verify footer (checksum) matches for the incoming file we are copying
- long checksum = CodecUtil.checkFooter(in);
-
- // this is poached from CodecUtil.writeFooter, but we need to use our own checksum, not
- // data.getChecksum(), but I think
- // adding a public method to CodecUtil to do that is somewhat dangerous:
- CodecUtil.writeBEInt(data, CodecUtil.FOOTER_MAGIC);
- CodecUtil.writeBEInt(data, 0);
- CodecUtil.writeBELong(data, checksum);
- }
- long endOffset = data.getFilePointer();
-
- long length = endOffset - startOffset;
-
- // write entry for file
- entries.writeString(IndexFileNames.stripSegmentName(file));
- entries.writeLong(startOffset);
- entries.writeLong(length);
- }
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWPostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWPostingsFormat.java
deleted file mode 100644
index fb042a1a9ad..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWPostingsFormat.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import java.io.IOException;
-import org.apache.lucene.backward_codecs.lucene40.blocktree.Lucene40BlockTreeTermsWriterV5;
-import org.apache.lucene.codecs.FieldsConsumer;
-import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.util.IOUtils;
-
-public class Lucene50RWPostingsFormat extends Lucene50PostingsFormat {
-
- @Override
- public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene50PostingsWriter(state);
- boolean success = false;
- try {
- FieldsConsumer ret =
- new Lucene40BlockTreeTermsWriterV5(
- state,
- postingsWriter,
- Lucene40BlockTreeTermsWriterV5.DEFAULT_MIN_BLOCK_SIZE,
- Lucene40BlockTreeTermsWriterV5.DEFAULT_MAX_BLOCK_SIZE);
- success = true;
- return ret;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(postingsWriter);
- }
- }
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWStoredFieldsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWStoredFieldsFormat.java
deleted file mode 100644
index 320819ae604..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWStoredFieldsFormat.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import java.io.IOException;
-import org.apache.lucene.backward_codecs.compressing.CompressionMode;
-import org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50RWCompressingStoredFieldsFormat;
-import org.apache.lucene.codecs.StoredFieldsFormat;
-import org.apache.lucene.codecs.StoredFieldsWriter;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-
-/** RW impersonation of Lucene50StoredFieldsFormat. */
-public final class Lucene50RWStoredFieldsFormat extends Lucene50StoredFieldsFormat {
-
- /** No-argument constructor. */
- public Lucene50RWStoredFieldsFormat() {
- super();
- }
-
- /** Constructor that takes a mode. */
- public Lucene50RWStoredFieldsFormat(Lucene50StoredFieldsFormat.Mode mode) {
- super(mode);
- }
-
- @Override
- public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context)
- throws IOException {
- String previous = si.putAttribute(MODE_KEY, mode.name());
- if (previous != null && previous.equals(mode.name()) == false) {
- throw new IllegalStateException(
- "found existing value for "
- + MODE_KEY
- + " for segment: "
- + si.name
- + "old="
- + previous
- + ", new="
- + mode.name());
- }
- return impl(mode).fieldsWriter(directory, si, context);
- }
-
- @Override
- StoredFieldsFormat impl(Mode mode) {
- switch (mode) {
- case BEST_SPEED:
- return new Lucene50RWCompressingStoredFieldsFormat(
- "Lucene50StoredFieldsFastData", Lucene50StoredFieldsFormat.FAST_MODE, 1 << 14, 128, 10);
- case BEST_COMPRESSION:
- return new Lucene50RWCompressingStoredFieldsFormat(
- "Lucene50StoredFieldsHighData", CompressionMode.HIGH_COMPRESSION, 61440, 512, 10);
- default:
- throw new AssertionError();
- }
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWTermVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWTermVectorsFormat.java
deleted file mode 100644
index d711a3f267c..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWTermVectorsFormat.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50RWCompressingTermVectorsFormat;
-
-/** RW impersonation of Lucene50StoredFieldsFormat. */
-public final class Lucene50RWTermVectorsFormat extends Lucene50RWCompressingTermVectorsFormat {
-
- /** Sole constructor. */
- public Lucene50RWTermVectorsFormat() {
- super("Lucene50TermVectorsData", "", Lucene50StoredFieldsFormat.FAST_MODE, 1 << 12, 128, 10);
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50SkipWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50SkipWriter.java
deleted file mode 100644
index f9b191c1e6f..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50SkipWriter.java
+++ /dev/null
@@ -1,237 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
-import org.apache.lucene.codecs.Impact;
-import org.apache.lucene.codecs.MultiLevelSkipListWriter;
-import org.apache.lucene.store.ByteBuffersDataOutput;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.IndexOutput;
-
-/**
- * Write skip lists with multiple levels, and support skip within block ints.
- *
- * <p>Assume that docFreq = 28, skipInterval = blockSize = 12
- *
- * <pre>
- * | block#0 | | block#1 | |vInts|
- * d d d d d d d d d d d d d d d d d d d d d d d d d d d d (posting list)
- * ^ ^ (level 0 skip point)
- * </pre>
- *
- * Note that skipWriter will ignore first document in block#0, since it is useless as a skip point.
- * Also, we'll never skip into the vInts block, only record skip data at the start its start
- * point(if it exist).
- *
- * <p>For each skip point, we will record: 1. docID in former position, i.e. for position 12, record
- * docID[11], etc. 2. its related file points(position, payload), 3. related numbers or
- * uptos(position, payload). 4. start offset.
- */
-final class Lucene50SkipWriter extends MultiLevelSkipListWriter {
- private int[] lastSkipDoc;
- private long[] lastSkipDocPointer;
- private long[] lastSkipPosPointer;
- private long[] lastSkipPayPointer;
- private int[] lastPayloadByteUpto;
-
- private final IndexOutput docOut;
- private final IndexOutput posOut;
- private final IndexOutput payOut;
-
- private int curDoc;
- private long curDocPointer;
- private long curPosPointer;
- private long curPayPointer;
- private int curPosBufferUpto;
- private int curPayloadByteUpto;
- private CompetitiveImpactAccumulator[] curCompetitiveFreqNorms;
- private boolean fieldHasPositions;
- private boolean fieldHasOffsets;
- private boolean fieldHasPayloads;
-
- public Lucene50SkipWriter(
- int maxSkipLevels,
- int blockSize,
- int docCount,
- IndexOutput docOut,
- IndexOutput posOut,
- IndexOutput payOut) {
- super(blockSize, 8, maxSkipLevels, docCount);
- this.docOut = docOut;
- this.posOut = posOut;
- this.payOut = payOut;
-
- lastSkipDoc = new int[maxSkipLevels];
- lastSkipDocPointer = new long[maxSkipLevels];
- if (posOut != null) {
- lastSkipPosPointer = new long[maxSkipLevels];
- if (payOut != null) {
- lastSkipPayPointer = new long[maxSkipLevels];
- }
- lastPayloadByteUpto = new int[maxSkipLevels];
- }
- curCompetitiveFreqNorms = new CompetitiveImpactAccumulator[maxSkipLevels];
- for (int i = 0; i < maxSkipLevels; ++i) {
- curCompetitiveFreqNorms[i] = new CompetitiveImpactAccumulator();
- }
- }
-
- public void setField(
- boolean fieldHasPositions, boolean fieldHasOffsets, boolean fieldHasPayloads) {
- this.fieldHasPositions = fieldHasPositions;
- this.fieldHasOffsets = fieldHasOffsets;
- this.fieldHasPayloads = fieldHasPayloads;
- }
-
- // tricky: we only skip data for blocks (terms with more than 128 docs), but re-init'ing the
- // skipper
- // is pretty slow for rare terms in large segments as we have to fill O(log #docs in segment) of
- // junk.
- // this is the vast majority of terms (worst case: ID field or similar). so in resetSkip() we
- // save
- // away the previous pointers, and lazy-init only if we need to buffer skip data for the term.
- private boolean initialized;
- long lastDocFP;
- long lastPosFP;
- long lastPayFP;
-
- @Override
- public void resetSkip() {
- lastDocFP = docOut.getFilePointer();
- if (fieldHasPositions) {
- lastPosFP = posOut.getFilePointer();
- if (fieldHasOffsets || fieldHasPayloads) {
- lastPayFP = payOut.getFilePointer();
- }
- }
- if (initialized) {
- for (CompetitiveImpactAccumulator acc : curCompetitiveFreqNorms) {
- acc.clear();
- }
- }
- initialized = false;
- }
-
- private void initSkip() {
- if (!initialized) {
- super.resetSkip();
- Arrays.fill(lastSkipDoc, 0);
- Arrays.fill(lastSkipDocPointer, lastDocFP);
- if (fieldHasPositions) {
- Arrays.fill(lastSkipPosPointer, lastPosFP);
- if (fieldHasPayloads) {
- Arrays.fill(lastPayloadByteUpto, 0);
- }
- if (fieldHasOffsets || fieldHasPayloads) {
- Arrays.fill(lastSkipPayPointer, lastPayFP);
- }
- }
- // sets of competitive freq,norm pairs should be empty at this point
- assert Arrays.stream(curCompetitiveFreqNorms)
- .map(CompetitiveImpactAccumulator::getCompetitiveFreqNormPairs)
- .mapToInt(Collection::size)
- .sum()
- == 0;
- initialized = true;
- }
- }
-
- /** Sets the values for the current skip data. */
- public void bufferSkip(
- int doc,
- CompetitiveImpactAccumulator competitiveFreqNorms,
- int numDocs,
- long posFP,
- long payFP,
- int posBufferUpto,
- int payloadByteUpto)
- throws IOException {
- initSkip();
- this.curDoc = doc;
- this.curDocPointer = docOut.getFilePointer();
- this.curPosPointer = posFP;
- this.curPayPointer = payFP;
- this.curPosBufferUpto = posBufferUpto;
- this.curPayloadByteUpto = payloadByteUpto;
- this.curCompetitiveFreqNorms[0].addAll(competitiveFreqNorms);
- bufferSkip(numDocs);
- }
-
- private final ByteBuffersDataOutput freqNormOut = ByteBuffersDataOutput.newResettableInstance();
-
- @Override
- protected void writeSkipData(int level, DataOutput skipBuffer) throws IOException {
-
- int delta = curDoc - lastSkipDoc[level];
-
- skipBuffer.writeVInt(delta);
- lastSkipDoc[level] = curDoc;
-
- skipBuffer.writeVLong(curDocPointer - lastSkipDocPointer[level]);
- lastSkipDocPointer[level] = curDocPointer;
-
- if (fieldHasPositions) {
-
- skipBuffer.writeVLong(curPosPointer - lastSkipPosPointer[level]);
- lastSkipPosPointer[level] = curPosPointer;
- skipBuffer.writeVInt(curPosBufferUpto);
-
- if (fieldHasPayloads) {
- skipBuffer.writeVInt(curPayloadByteUpto);
- }
-
- if (fieldHasOffsets || fieldHasPayloads) {
- skipBuffer.writeVLong(curPayPointer - lastSkipPayPointer[level]);
- lastSkipPayPointer[level] = curPayPointer;
- }
- }
-
- CompetitiveImpactAccumulator competitiveFreqNorms = curCompetitiveFreqNorms[level];
- assert competitiveFreqNorms.getCompetitiveFreqNormPairs().size() > 0;
- if (level + 1 < numberOfSkipLevels) {
- curCompetitiveFreqNorms[level + 1].addAll(competitiveFreqNorms);
- }
- writeImpacts(competitiveFreqNorms, freqNormOut);
- skipBuffer.writeVInt(Math.toIntExact(freqNormOut.size()));
- freqNormOut.copyTo(skipBuffer);
- freqNormOut.reset();
- competitiveFreqNorms.clear();
- }
-
- static void writeImpacts(CompetitiveImpactAccumulator acc, DataOutput out) throws IOException {
- Collection<Impact> impacts = acc.getCompetitiveFreqNormPairs();
- Impact previous = new Impact(0, 0);
- for (Impact impact : impacts) {
- assert impact.freq > previous.freq;
- assert Long.compareUnsigned(impact.norm, previous.norm) > 0;
- int freqDelta = impact.freq - previous.freq - 1;
- long normDelta = impact.norm - previous.norm - 1;
- if (normDelta == 0) {
- // most of time, norm only increases by 1, so we can fold everything in a single byte
- out.writeVInt(freqDelta << 1);
- } else {
- out.writeVInt((freqDelta << 1) | 1);
- out.writeZLong(normDelta);
- }
- previous = impact;
- }
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestBlockPostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestBlockPostingsFormat.java
deleted file mode 100644
index 4205b5d70e5..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestBlockPostingsFormat.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import org.apache.lucene.backward_codecs.lucene40.blocktree.FieldReader;
-import org.apache.lucene.backward_codecs.lucene40.blocktree.Stats;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
-import org.apache.lucene.codecs.Impact;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.FreqAndNormBuffer;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.tests.analysis.MockAnalyzer;
-import org.apache.lucene.tests.index.BasePostingsFormatTestCase;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-import org.apache.lucene.tests.util.TestUtil;
-
-/** Tests BlockPostingsFormat */
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestBlockPostingsFormat extends BasePostingsFormatTestCase {
- private final Codec codec = TestUtil.alwaysPostingsFormat(new Lucene50RWPostingsFormat());
-
- @Override
- protected Codec getCodec() {
- return codec;
- }
-
- /** Make sure the final sub-block(s) are not skipped. */
- public void testFinalBlock() throws Exception {
- Directory d = newDirectory();
- IndexWriter w = new IndexWriter(d, new IndexWriterConfig(new MockAnalyzer(random())));
- for (int i = 0; i < 25; i++) {
- Document doc = new Document();
- doc.add(newStringField("field", Character.toString((char) (97 + i)), Field.Store.NO));
- doc.add(newStringField("field", "z" + Character.toString((char) (97 + i)), Field.Store.NO));
- w.addDocument(doc);
- }
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- assertEquals(1, r.leaves().size());
- FieldReader field = (FieldReader) r.leaves().get(0).reader().terms("field");
- // We should see exactly two blocks: one root block (prefix empty string) and one block for z*
- // terms (prefix z):
- Stats stats = field.getStats();
- assertEquals(0, stats.floorBlockCount);
- assertEquals(2, stats.nonFloorBlockCount);
- r.close();
- w.close();
- d.close();
- }
-
- public void testImpactSerialization() throws IOException {
- // omit norms and omit freqs
- doTestImpactSerialization(Collections.singletonList(new Impact(1, 1L)));
-
- // omit freqs
- doTestImpactSerialization(Collections.singletonList(new Impact(1, 42L)));
- // omit freqs with very large norms
- doTestImpactSerialization(Collections.singletonList(new Impact(1, -100L)));
-
- // omit norms
- doTestImpactSerialization(Collections.singletonList(new Impact(30, 1L)));
- // omit norms with large freq
- doTestImpactSerialization(Collections.singletonList(new Impact(500, 1L)));
-
- // freqs and norms, basic
- doTestImpactSerialization(
- Arrays.asList(
- new Impact(1, 7L),
- new Impact(3, 9L),
- new Impact(7, 10L),
- new Impact(15, 11L),
- new Impact(20, 13L),
- new Impact(28, 14L)));
-
- // freqs and norms, high values
- doTestImpactSerialization(
- Arrays.asList(
- new Impact(2, 2L),
- new Impact(10, 10L),
- new Impact(12, 50L),
- new Impact(50, -100L),
- new Impact(1000, -80L),
- new Impact(1005, -3L)));
- }
-
- private void doTestImpactSerialization(List<Impact> impacts) throws IOException {
- CompetitiveImpactAccumulator acc = new CompetitiveImpactAccumulator();
- for (Impact impact : impacts) {
- acc.add(impact.freq, impact.norm);
- }
- try (Directory dir = newDirectory()) {
- try (IndexOutput out = EndiannessReverserUtil.createOutput(dir, "foo", IOContext.DEFAULT)) {
- Lucene50SkipWriter.writeImpacts(acc, out);
- }
- try (IndexInput in = EndiannessReverserUtil.openInput(dir, "foo", IOContext.DEFAULT)) {
- byte[] b = new byte[Math.toIntExact(in.length())];
- in.readBytes(b, 0, b.length);
- FreqAndNormBuffer impacts2 =
- Lucene50ScoreSkipReader.readImpacts(new ByteArrayDataInput(b), new FreqAndNormBuffer());
- List<Impact> impacts2AsList = new ArrayList<>();
- for (int i = 0; i < impacts2.size; ++i) {
- impacts2AsList.add(new Impact(impacts2.freqs[i], impacts2.norms[i]));
- }
- assertEquals(impacts, impacts2AsList);
- }
- }
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestBlockPostingsFormat2.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestBlockPostingsFormat2.java
deleted file mode 100644
index 337ec450c43..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestBlockPostingsFormat2.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.tests.analysis.MockAnalyzer;
-import org.apache.lucene.tests.index.RandomIndexWriter;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-import org.apache.lucene.tests.util.TestUtil;
-
-/** Tests special cases of BlockPostingsFormat */
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestBlockPostingsFormat2 extends LuceneTestCase {
- Directory dir;
- RandomIndexWriter iw;
-
- @Override
- public void setUp() throws Exception {
- super.setUp();
- dir = newFSDirectory(createTempDir("testDFBlockSize"));
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setCodec(TestUtil.alwaysPostingsFormat(new Lucene50RWPostingsFormat()));
- iw = new RandomIndexWriter(random(), dir, iwc);
- iw.setDoRandomForceMerge(false); // we will ourselves
- }
-
- @Override
- public void tearDown() throws Exception {
- iw.close();
- TestUtil.checkIndex(dir); // for some extra coverage, checkIndex before we forceMerge
- IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
- iwc.setCodec(TestUtil.alwaysPostingsFormat(new Lucene50RWPostingsFormat()));
- iwc.setOpenMode(OpenMode.APPEND);
- IndexWriter iw = new IndexWriter(dir, iwc);
- iw.forceMerge(1);
- iw.close();
- dir.close(); // just force a checkindex for now
- super.tearDown();
- }
-
- private Document newDocument() {
- Document doc = new Document();
- for (IndexOptions option : IndexOptions.values()) {
- if (option == IndexOptions.NONE) {
- continue;
- }
- FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
- // turn on tvs for a cross-check, since we rely upon checkindex in this test (for now)
- ft.setStoreTermVectors(true);
- ft.setStoreTermVectorOffsets(true);
- ft.setStoreTermVectorPositions(true);
- ft.setStoreTermVectorPayloads(true);
- ft.setIndexOptions(option);
- doc.add(new Field(option.toString(), "", ft));
- }
- return doc;
- }
-
- /** tests terms with df = blocksize */
- public void testDFBlockSize() throws Exception {
- Document doc = newDocument();
- for (int i = 0; i < Lucene50PostingsFormat.BLOCK_SIZE; i++) {
- for (IndexableField f : doc.getFields()) {
- ((Field) f).setStringValue(f.name() + " " + f.name() + "_2");
- }
- iw.addDocument(doc);
- }
- }
-
- /** tests terms with df % blocksize = 0 */
- public void testDFBlockSizeMultiple() throws Exception {
- Document doc = newDocument();
- for (int i = 0; i < Lucene50PostingsFormat.BLOCK_SIZE * 16; i++) {
- for (IndexableField f : doc.getFields()) {
- ((Field) f).setStringValue(f.name() + " " + f.name() + "_2");
- }
- iw.addDocument(doc);
- }
- }
-
- /** tests terms with ttf = blocksize */
- public void testTTFBlockSize() throws Exception {
- Document doc = newDocument();
- for (int i = 0; i < Lucene50PostingsFormat.BLOCK_SIZE / 2; i++) {
- for (IndexableField f : doc.getFields()) {
- ((Field) f)
- .setStringValue(f.name() + " " + f.name() + " " + f.name() + "_2 " + f.name() + "_2");
- }
- iw.addDocument(doc);
- }
- }
-
- /** tests terms with ttf % blocksize = 0 */
- public void testTTFBlockSizeMultiple() throws Exception {
- Document doc = newDocument();
- for (int i = 0; i < Lucene50PostingsFormat.BLOCK_SIZE / 2; i++) {
- for (IndexableField f : doc.getFields()) {
- String proto =
- (f.name() + " " + f.name() + " " + f.name() + " " + f.name() + " " + f.name() + "_2 "
- + f.name() + "_2 " + f.name() + "_2 " + f.name() + "_2");
- StringBuilder val = new StringBuilder();
- for (int j = 0; j < 16; j++) {
- val.append(proto);
- val.append(" ");
- }
- ((Field) f).setStringValue(val.toString());
- }
- iw.addDocument(doc);
- }
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestBlockPostingsFormat3.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestBlockPostingsFormat3.java
deleted file mode 100644
index 84f6ac950e1..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestBlockPostingsFormat3.java
+++ /dev/null
@@ -1,493 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Random;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.TokenFilter;
-import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.TermsEnum.SeekStatus;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.tests.analysis.MockFixedLengthPayloadFilter;
-import org.apache.lucene.tests.analysis.MockTokenizer;
-import org.apache.lucene.tests.analysis.MockVariableLengthPayloadFilter;
-import org.apache.lucene.tests.index.RandomIndexWriter;
-import org.apache.lucene.tests.util.English;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-import org.apache.lucene.tests.util.TestUtil;
-import org.apache.lucene.tests.util.automaton.AutomatonTestUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.CompiledAutomaton;
-import org.apache.lucene.util.automaton.Operations;
-import org.apache.lucene.util.automaton.RegExp;
-
-/** Tests partial enumeration (only pulling a subset of the indexed data) */
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestBlockPostingsFormat3 extends LuceneTestCase {
- private final int MAXDOC =
- TEST_NIGHTLY ? Lucene50PostingsFormat.BLOCK_SIZE * 20 : Lucene50PostingsFormat.BLOCK_SIZE * 3;
-
- // creates 8 fields with different options and does "duels" of fields against each other
- public void test() throws Exception {
- Directory dir = newDirectory();
- Analyzer analyzer =
- new Analyzer(Analyzer.PER_FIELD_REUSE_STRATEGY) {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new MockTokenizer();
- if (fieldName.contains("payloadsFixed")) {
- TokenFilter filter = new MockFixedLengthPayloadFilter(new Random(0), tokenizer, 1);
- return new TokenStreamComponents(tokenizer, filter);
- } else if (fieldName.contains("payloadsVariable")) {
- TokenFilter filter = new MockVariableLengthPayloadFilter(new Random(0), tokenizer);
- return new TokenStreamComponents(tokenizer, filter);
- } else {
- return new TokenStreamComponents(tokenizer);
- }
- }
- };
- IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
- iwc.setCodec(TestUtil.alwaysPostingsFormat(new Lucene50RWPostingsFormat()));
- // TODO we could actually add more fields implemented with different PFs
- // or, just put this test into the usual rotation?
- RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
- Document doc = new Document();
- FieldType docsOnlyType = new FieldType(TextField.TYPE_NOT_STORED);
- // turn this on for a cross-check
- docsOnlyType.setStoreTermVectors(true);
- docsOnlyType.setIndexOptions(IndexOptions.DOCS);
-
- FieldType docsAndFreqsType = new FieldType(TextField.TYPE_NOT_STORED);
- // turn this on for a cross-check
- docsAndFreqsType.setStoreTermVectors(true);
- docsAndFreqsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
-
- FieldType positionsType = new FieldType(TextField.TYPE_NOT_STORED);
- // turn these on for a cross-check
- positionsType.setStoreTermVectors(true);
- positionsType.setStoreTermVectorPositions(true);
- positionsType.setStoreTermVectorOffsets(true);
- positionsType.setStoreTermVectorPayloads(true);
- FieldType offsetsType = new FieldType(positionsType);
- offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- Field field1 = new Field("field1docs", "", docsOnlyType);
- Field field2 = new Field("field2freqs", "", docsAndFreqsType);
- Field field3 = new Field("field3positions", "", positionsType);
- Field field4 = new Field("field4offsets", "", offsetsType);
- Field field5 = new Field("field5payloadsFixed", "", positionsType);
- Field field6 = new Field("field6payloadsVariable", "", positionsType);
- Field field7 = new Field("field7payloadsFixedOffsets", "", offsetsType);
- Field field8 = new Field("field8payloadsVariableOffsets", "", offsetsType);
- doc.add(field1);
- doc.add(field2);
- doc.add(field3);
- doc.add(field4);
- doc.add(field5);
- doc.add(field6);
- doc.add(field7);
- doc.add(field8);
- for (int i = 0; i < MAXDOC; i++) {
- String stringValue =
- Integer.toString(i)
- + " verycommon "
- + English.intToEnglish(i).replace('-', ' ')
- + " "
- + TestUtil.randomSimpleString(random());
- field1.setStringValue(stringValue);
- field2.setStringValue(stringValue);
- field3.setStringValue(stringValue);
- field4.setStringValue(stringValue);
- field5.setStringValue(stringValue);
- field6.setStringValue(stringValue);
- field7.setStringValue(stringValue);
- field8.setStringValue(stringValue);
- iw.addDocument(doc);
- }
- iw.close();
- verify(dir);
- TestUtil.checkIndex(dir); // for some extra coverage, checkIndex before we forceMerge
- iwc = newIndexWriterConfig(analyzer);
- iwc.setCodec(TestUtil.alwaysPostingsFormat(new Lucene50RWPostingsFormat()));
- iwc.setOpenMode(OpenMode.APPEND);
- IndexWriter iw2 = new IndexWriter(dir, iwc);
- iw2.forceMerge(1);
- iw2.close();
- verify(dir);
- dir.close();
- }
-
- private void verify(Directory dir) throws Exception {
- DirectoryReader ir = DirectoryReader.open(dir);
- for (LeafReaderContext leaf : ir.leaves()) {
- LeafReader leafReader = leaf.reader();
- assertTerms(leafReader.terms("field1docs"), leafReader.terms("field2freqs"), true);
- assertTerms(leafReader.terms("field3positions"), leafReader.terms("field4offsets"), true);
- assertTerms(leafReader.terms("field4offsets"), leafReader.terms("field5payloadsFixed"), true);
- assertTerms(
- leafReader.terms("field5payloadsFixed"),
- leafReader.terms("field6payloadsVariable"),
- true);
- assertTerms(
- leafReader.terms("field6payloadsVariable"),
- leafReader.terms("field7payloadsFixedOffsets"),
- true);
- assertTerms(
- leafReader.terms("field7payloadsFixedOffsets"),
- leafReader.terms("field8payloadsVariableOffsets"),
- true);
- }
- ir.close();
- }
-
- // following code is almost an exact dup of code from TestDuelingCodecs: sorry!
-
- public void assertTerms(Terms leftTerms, Terms rightTerms, boolean deep) throws Exception {
- if (leftTerms == null || rightTerms == null) {
- assertNull(leftTerms);
- assertNull(rightTerms);
- return;
- }
- assertTermsStatistics(leftTerms, rightTerms);
-
- // NOTE: we don't assert hasOffsets/hasPositions/hasPayloads because they are allowed to be
- // different
-
- boolean bothHaveFreqs = leftTerms.hasFreqs() && rightTerms.hasFreqs();
- boolean bothHavePositions = leftTerms.hasPositions() && rightTerms.hasPositions();
- TermsEnum leftTermsEnum = leftTerms.iterator();
- TermsEnum rightTermsEnum = rightTerms.iterator();
- assertTermsEnum(leftTermsEnum, rightTermsEnum, true, bothHaveFreqs, bothHavePositions);
-
- assertTermsSeeking(leftTerms, rightTerms);
-
- if (deep) {
- int numIntersections = atLeast(3);
- for (int i = 0; i < numIntersections; i++) {
- String re = AutomatonTestUtil.randomRegexp(random());
- Automaton a = new RegExp(re, RegExp.NONE).toAutomaton();
- a = Operations.determinize(a, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
- CompiledAutomaton automaton = new CompiledAutomaton(a);
- if (automaton.type == CompiledAutomaton.AUTOMATON_TYPE.NORMAL) {
- // TODO: test start term too
- TermsEnum leftIntersection = leftTerms.intersect(automaton, null);
- TermsEnum rightIntersection = rightTerms.intersect(automaton, null);
- assertTermsEnum(
- leftIntersection, rightIntersection, rarely(), bothHaveFreqs, bothHavePositions);
- }
- }
- }
- }
-
- private void assertTermsSeeking(Terms leftTerms, Terms rightTerms) throws Exception {
- TermsEnum leftEnum = null;
- TermsEnum rightEnum = null;
-
- // just an upper bound
- int numTests = atLeast(20);
- Random random = random();
-
- // collect this number of terms from the left side
- HashSet<BytesRef> tests = new HashSet<>();
- int numPasses = 0;
- while (numPasses < 10 && tests.size() < numTests) {
- leftEnum = leftTerms.iterator();
- BytesRef term = null;
- while ((term = leftEnum.next()) != null) {
- int code = random.nextInt(10);
- if (code == 0) {
- // the term
- tests.add(BytesRef.deepCopyOf(term));
- } else if (code == 1) {
- // truncated subsequence of term
- term = BytesRef.deepCopyOf(term);
- if (term.length > 0) {
- // truncate it
- term.length = random.nextInt(term.length);
- }
- } else if (code == 2) {
- // term, but ensure a non-zero offset
- byte[] newbytes = new byte[term.length + 5];
- System.arraycopy(term.bytes, term.offset, newbytes, 5, term.length);
- tests.add(new BytesRef(newbytes, 5, term.length));
- }
- }
- numPasses++;
- }
-
- ArrayList<BytesRef> shuffledTests = new ArrayList<>(tests);
- Collections.shuffle(shuffledTests, random);
-
- for (BytesRef b : shuffledTests) {
- leftEnum = leftTerms.iterator();
- rightEnum = rightTerms.iterator();
-
- assertEquals(leftEnum.seekExact(b), rightEnum.seekExact(b));
- assertEquals(leftEnum.seekExact(b), rightEnum.seekExact(b));
-
- SeekStatus leftStatus;
- SeekStatus rightStatus;
-
- leftStatus = leftEnum.seekCeil(b);
- rightStatus = rightEnum.seekCeil(b);
- assertEquals(leftStatus, rightStatus);
- if (leftStatus != SeekStatus.END) {
- assertEquals(leftEnum.term(), rightEnum.term());
- }
-
- leftStatus = leftEnum.seekCeil(b);
- rightStatus = rightEnum.seekCeil(b);
- assertEquals(leftStatus, rightStatus);
- if (leftStatus != SeekStatus.END) {
- assertEquals(leftEnum.term(), rightEnum.term());
- }
- }
- }
-
- /** checks collection-level statistics on Terms */
- public void assertTermsStatistics(Terms leftTerms, Terms rightTerms) throws Exception {
- assertEquals(leftTerms.getDocCount(), rightTerms.getDocCount());
- assertEquals(leftTerms.getSumDocFreq(), rightTerms.getSumDocFreq());
- if (leftTerms.hasFreqs() && rightTerms.hasFreqs()) {
- assertEquals(leftTerms.getSumTotalTermFreq(), rightTerms.getSumTotalTermFreq());
- }
- if (leftTerms.size() != -1 && rightTerms.size() != -1) {
- assertEquals(leftTerms.size(), rightTerms.size());
- }
- }
-
- /**
- * checks the terms enum sequentially if deep is false, it does a 'shallow' test that doesnt go
- * down to the docsenums
- */
- public void assertTermsEnum(
- TermsEnum leftTermsEnum,
- TermsEnum rightTermsEnum,
- boolean deep,
- boolean hasFreqs,
- boolean hasPositions)
- throws Exception {
- BytesRef term;
- PostingsEnum leftPositions = null;
- PostingsEnum rightPositions = null;
- PostingsEnum leftDocs = null;
- PostingsEnum rightDocs = null;
-
- while ((term = leftTermsEnum.next()) != null) {
- assertEquals(term, rightTermsEnum.next());
- assertTermStats(leftTermsEnum, rightTermsEnum, hasFreqs);
- if (deep) {
- if (hasPositions) {
- // with payloads + off
- assertDocsAndPositionsEnum(
- leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.ALL),
- rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.ALL));
-
- assertPositionsSkipping(
- leftTermsEnum.docFreq(),
- leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.ALL),
- rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.ALL));
- // with payloads only
- assertDocsAndPositionsEnum(
- leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.PAYLOADS),
- rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.PAYLOADS));
-
- assertPositionsSkipping(
- leftTermsEnum.docFreq(),
- leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.PAYLOADS),
- rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.PAYLOADS));
-
- // with offsets only
- assertDocsAndPositionsEnum(
- leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.OFFSETS),
- rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.OFFSETS));
-
- assertPositionsSkipping(
- leftTermsEnum.docFreq(),
- leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.OFFSETS),
- rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.OFFSETS));
-
- // with positions only
- assertDocsAndPositionsEnum(
- leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.POSITIONS),
- rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.POSITIONS));
-
- assertPositionsSkipping(
- leftTermsEnum.docFreq(),
- leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.POSITIONS),
- rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.POSITIONS));
- }
-
- // with freqs:
- assertDocsEnum(
- leftDocs = leftTermsEnum.postings(leftDocs),
- rightDocs = rightTermsEnum.postings(rightDocs));
-
- // w/o freqs:
- assertDocsEnum(
- leftDocs = leftTermsEnum.postings(leftDocs, PostingsEnum.NONE),
- rightDocs = rightTermsEnum.postings(rightDocs, PostingsEnum.NONE));
-
- // with freqs:
- assertDocsSkipping(
- leftTermsEnum.docFreq(),
- leftDocs = leftTermsEnum.postings(leftDocs),
- rightDocs = rightTermsEnum.postings(rightDocs));
-
- // w/o freqs:
- assertDocsSkipping(
- leftTermsEnum.docFreq(),
- leftDocs = leftTermsEnum.postings(leftDocs, PostingsEnum.NONE),
- rightDocs = rightTermsEnum.postings(rightDocs, PostingsEnum.NONE));
- }
- }
- assertNull(rightTermsEnum.next());
- }
-
- /** checks term-level statistics */
- public void assertTermStats(
- TermsEnum leftTermsEnum, TermsEnum rightTermsEnum, boolean bothHaveFreqs) throws Exception {
- assertEquals(leftTermsEnum.docFreq(), rightTermsEnum.docFreq());
- if (bothHaveFreqs) {
- assertEquals(leftTermsEnum.totalTermFreq(), rightTermsEnum.totalTermFreq());
- }
- }
-
- /** checks docs + freqs + positions + payloads, sequentially */
- public void assertDocsAndPositionsEnum(PostingsEnum leftDocs, PostingsEnum rightDocs)
- throws Exception {
- assertNotNull(leftDocs);
- assertNotNull(rightDocs);
- assertEquals(-1, leftDocs.docID());
- assertEquals(-1, rightDocs.docID());
- int docid;
- while ((docid = leftDocs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- assertEquals(docid, rightDocs.nextDoc());
- int freq = leftDocs.freq();
- assertEquals(freq, rightDocs.freq());
- for (int i = 0; i < freq; i++) {
- assertEquals(leftDocs.nextPosition(), rightDocs.nextPosition());
- // we don't assert offsets/payloads, they are allowed to be different
- }
- }
- assertEquals(DocIdSetIterator.NO_MORE_DOCS, rightDocs.nextDoc());
- }
-
- /** checks docs + freqs, sequentially */
- public void assertDocsEnum(PostingsEnum leftDocs, PostingsEnum rightDocs) throws Exception {
- if (leftDocs == null) {
- assertNull(rightDocs);
- return;
- }
- assertEquals(-1, leftDocs.docID());
- assertEquals(-1, rightDocs.docID());
- int docid;
- while ((docid = leftDocs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- assertEquals(docid, rightDocs.nextDoc());
- // we don't assert freqs, they are allowed to be different
- }
- assertEquals(DocIdSetIterator.NO_MORE_DOCS, rightDocs.nextDoc());
- }
-
- /** checks advancing docs */
- public void assertDocsSkipping(int docFreq, PostingsEnum leftDocs, PostingsEnum rightDocs)
- throws Exception {
- if (leftDocs == null) {
- assertNull(rightDocs);
- return;
- }
- int docid = -1;
- int averageGap = MAXDOC / (1 + docFreq);
- int skipInterval = 16;
-
- while (true) {
- if (random().nextBoolean()) {
- // nextDoc()
- docid = leftDocs.nextDoc();
- assertEquals(docid, rightDocs.nextDoc());
- } else {
- // advance()
- int skip =
- docid + (int) Math.ceil(Math.abs(skipInterval + random().nextGaussian() * averageGap));
- docid = leftDocs.advance(skip);
- assertEquals(docid, rightDocs.advance(skip));
- }
-
- if (docid == DocIdSetIterator.NO_MORE_DOCS) {
- return;
- }
- // we don't assert freqs, they are allowed to be different
- }
- }
-
- /** checks advancing docs + positions */
- public void assertPositionsSkipping(int docFreq, PostingsEnum leftDocs, PostingsEnum rightDocs)
- throws Exception {
- if (leftDocs == null || rightDocs == null) {
- assertNull(leftDocs);
- assertNull(rightDocs);
- return;
- }
-
- int docid = -1;
- int averageGap = MAXDOC / (1 + docFreq);
- int skipInterval = 16;
-
- while (true) {
- if (random().nextBoolean()) {
- // nextDoc()
- docid = leftDocs.nextDoc();
- assertEquals(docid, rightDocs.nextDoc());
- } else {
- // advance()
- int skip =
- docid + (int) Math.ceil(Math.abs(skipInterval + random().nextGaussian() * averageGap));
- docid = leftDocs.advance(skip);
- assertEquals(docid, rightDocs.advance(skip));
- }
-
- if (docid == DocIdSetIterator.NO_MORE_DOCS) {
- return;
- }
- int freq = leftDocs.freq();
- assertEquals(freq, rightDocs.freq());
- for (int i = 0; i < freq; i++) {
- assertEquals(leftDocs.nextPosition(), rightDocs.nextPosition());
- // we don't compare the payloads, it's allowed that one is empty etc
- }
- }
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestForUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestForUtil.java
deleted file mode 100644
index 6c9bfebc20b..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestForUtil.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import static org.apache.lucene.backward_codecs.lucene50.ForUtil.MAX_DATA_SIZE;
-import static org.apache.lucene.backward_codecs.lucene50.ForUtil.MAX_ENCODED_SIZE;
-import static org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat.BLOCK_SIZE;
-
-import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
-import java.io.IOException;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.store.ByteBuffersDirectory;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.packed.PackedInts;
-
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestForUtil extends LuceneTestCase {
-
- public void testEncodeDecode() throws IOException {
- final int iterations = RandomNumbers.randomIntBetween(random(), 1, 1000);
- final float acceptableOverheadRatio = random().nextFloat();
- final int[] values = new int[iterations * BLOCK_SIZE];
- for (int i = 0; i < iterations; ++i) {
- final int bpv = random().nextInt(32);
- if (bpv == 0) {
- final int value = RandomNumbers.randomIntBetween(random(), 0, Integer.MAX_VALUE);
- for (int j = 0; j < BLOCK_SIZE; ++j) {
- values[i * BLOCK_SIZE + j] = value;
- }
- } else {
- for (int j = 0; j < BLOCK_SIZE; ++j) {
- values[i * BLOCK_SIZE + j] =
- RandomNumbers.randomIntBetween(random(), 0, (int) PackedInts.maxValue(bpv));
- }
- }
- }
-
- final Directory d = new ByteBuffersDirectory();
- final long endPointer;
-
- {
- // encode
- IndexOutput out = EndiannessReverserUtil.createOutput(d, "test.bin", IOContext.DEFAULT);
- final ForUtil forUtil = new ForUtil(acceptableOverheadRatio, out);
-
- for (int i = 0; i < iterations; ++i) {
- // Although values after BLOCK_SIZE are garbage, we need to allocate extra bytes to avoid
- // AIOOBE.
- int[] block =
- ArrayUtil.grow(ArrayUtil.copyOfSubArray(values, i * BLOCK_SIZE, (i + 1) * BLOCK_SIZE));
- forUtil.writeBlock(ArrayUtil.grow(block, MAX_DATA_SIZE), new byte[MAX_ENCODED_SIZE], out);
- }
- endPointer = out.getFilePointer();
- out.close();
- }
-
- {
- // decode
- IndexInput in = EndiannessReverserUtil.openInput(d, "test.bin", IOContext.READONCE);
- final ForUtil forUtil = new ForUtil(in);
- for (int i = 0; i < iterations; ++i) {
- if (random().nextBoolean()) {
- forUtil.skipBlock(in);
- continue;
- }
- final int[] restored = new int[MAX_DATA_SIZE];
- forUtil.readBlock(in, new byte[MAX_ENCODED_SIZE], restored);
- assertArrayEquals(
- ArrayUtil.copyOfSubArray(values, i * BLOCK_SIZE, (i + 1) * BLOCK_SIZE),
- ArrayUtil.copyOfSubArray(restored, 0, BLOCK_SIZE));
- }
- assertEquals(endPointer, in.getFilePointer());
- in.close();
- }
-
- d.close();
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50CompoundFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50CompoundFormat.java
deleted file mode 100644
index 69c38210c7c..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50CompoundFormat.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import org.apache.lucene.backward_codecs.lucene87.Lucene87RWCodec;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.tests.index.BaseCompoundFormatTestCase;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestLucene50CompoundFormat extends BaseCompoundFormatTestCase {
-
- @Override
- protected Codec getCodec() {
- return new Lucene87RWCodec();
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50LiveDocsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50LiveDocsFormat.java
deleted file mode 100644
index fca261f2a8a..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50LiveDocsFormat.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import org.apache.lucene.backward_codecs.lucene86.Lucene86RWCodec;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.tests.index.BaseLiveDocsFormatTestCase;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestLucene50LiveDocsFormat extends BaseLiveDocsFormatTestCase {
-
- @Override
- protected Codec getCodec() {
- return new Lucene86RWCodec();
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50StoredFieldsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50StoredFieldsFormat.java
deleted file mode 100644
index 51fd2a6c50d..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50StoredFieldsFormat.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import org.apache.lucene.backward_codecs.lucene86.Lucene86RWCodec;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestLucene50StoredFieldsFormat extends BaseStoredFieldsFormatTestCase {
- @Override
- protected Codec getCodec() {
- return new Lucene86RWCodec();
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50StoredFieldsFormatHighCompression.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50StoredFieldsFormatHighCompression.java
deleted file mode 100644
index 4089a28e977..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50StoredFieldsFormatHighCompression.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat.Mode;
-import org.apache.lucene.backward_codecs.lucene86.Lucene86RWCodec;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.StoredField;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.StoredFields;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestLucene50StoredFieldsFormatHighCompression extends BaseStoredFieldsFormatTestCase {
- @Override
- protected Codec getCodec() {
- return new Lucene86RWCodec(Mode.BEST_COMPRESSION);
- }
-
- /**
- * Change compression params (leaving it the same for old segments) and tests that nothing breaks.
- */
- public void testMixedCompressions() throws Exception {
- Directory dir = newDirectory();
- for (int i = 0; i < 10; i++) {
- IndexWriterConfig iwc = newIndexWriterConfig();
- iwc.setCodec(new Lucene86RWCodec(RandomPicks.randomFrom(random(), Mode.values())));
- IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig());
- Document doc = new Document();
- doc.add(new StoredField("field1", "value1"));
- doc.add(new StoredField("field2", "value2"));
- iw.addDocument(doc);
- if (random().nextInt(4) == 0) {
- iw.forceMerge(1);
- }
- iw.commit();
- iw.close();
- }
-
- DirectoryReader ir = DirectoryReader.open(dir);
- assertEquals(10, ir.numDocs());
- StoredFields storedFields = ir.storedFields();
- for (int i = 0; i < 10; i++) {
- Document doc = storedFields.document(i);
- assertEquals("value1", doc.get("field1"));
- assertEquals("value2", doc.get("field2"));
- }
- ir.close();
- // checkindex
- dir.close();
- }
-
- public void testInvalidOptions() {
- expectThrows(
- NullPointerException.class,
- () -> {
- new Lucene86RWCodec(null);
- });
-
- expectThrows(
- NullPointerException.class,
- () -> {
- new Lucene50StoredFieldsFormat(null);
- });
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50StoredFieldsFormatMergeInstance.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50StoredFieldsFormatMergeInstance.java
deleted file mode 100644
index 5034e4032d6..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50StoredFieldsFormatMergeInstance.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-
-/** Test the merge instance of the Lucene50 stored fields format. */
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestLucene50StoredFieldsFormatMergeInstance extends TestLucene50StoredFieldsFormat {
-
- @Override
- protected boolean shouldTestMergeInstance() {
- return true;
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50TermVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50TermVectorsFormat.java
deleted file mode 100644
index 388707407c8..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50TermVectorsFormat.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50;
-
-import org.apache.lucene.backward_codecs.lucene87.Lucene87RWCodec;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.tests.index.BaseTermVectorsFormatTestCase;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestLucene50TermVectorsFormat extends BaseTermVectorsFormatTestCase {
-
- @Override
- protected Codec getCodec() {
- return new Lucene87RWCodec();
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndexWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndexWriter.java
deleted file mode 100644
index 857505fd1f2..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndexWriter.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50.compressing;
-
-import static org.apache.lucene.backward_codecs.lucene50.compressing.FieldsIndexReader.VERSION_CURRENT;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.lucene.backward_codecs.packed.LegacyDirectMonotonicReader;
-import org.apache.lucene.backward_codecs.packed.LegacyDirectMonotonicWriter;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.IOUtils;
-
-/**
- * Efficient index format for block-based {@link Codec}s.
- *
- * <p>For each block of compressed stored fields, this stores the first document of the block and
- * the start pointer of the block in a {@link LegacyDirectMonotonicWriter}. At read time, the docID
- * is binary-searched in the {@link LegacyDirectMonotonicWriter} that records doc IDS, and the
- * returned index is used to look up the start pointer in the {@link LegacyDirectMonotonicReader}
- * that records start pointers.
- *
- * @lucene.internal
- */
-public final class FieldsIndexWriter implements Closeable {
-
- private final Directory dir;
- private final String name;
- private final String suffix;
- private final String extension;
- private final String codecName;
- private final byte[] id;
- private final int blockShift;
- private final IOContext ioContext;
- private IndexOutput docsOut;
- private IndexOutput filePointersOut;
- private int totalDocs;
- private int totalChunks;
- private long previousFP;
-
- FieldsIndexWriter(
- Directory dir,
- String name,
- String suffix,
- String extension,
- String codecName,
- byte[] id,
- int blockShift,
- IOContext ioContext)
- throws IOException {
- this.dir = dir;
- this.name = name;
- this.suffix = suffix;
- this.extension = extension;
- this.codecName = codecName;
- this.id = id;
- this.blockShift = blockShift;
- this.ioContext = ioContext;
- this.docsOut =
- EndiannessReverserUtil.createTempOutput(dir, name, codecName + "-doc_ids", ioContext);
- boolean success = false;
- try {
- CodecUtil.writeHeader(docsOut, codecName + "Docs", VERSION_CURRENT);
- filePointersOut =
- EndiannessReverserUtil.createTempOutput(
- dir, name, codecName + "file_pointers", ioContext);
- CodecUtil.writeHeader(filePointersOut, codecName + "FilePointers", VERSION_CURRENT);
- success = true;
- } finally {
- if (success == false) {
- close();
- }
- }
- }
-
- void writeIndex(int numDocs, long startPointer) throws IOException {
- assert startPointer >= previousFP;
- docsOut.writeVInt(numDocs);
- filePointersOut.writeVLong(startPointer - previousFP);
- previousFP = startPointer;
- totalDocs += numDocs;
- totalChunks++;
- }
-
- void finish(int numDocs, long maxPointer, IndexOutput metaOut) throws IOException {
- if (numDocs != totalDocs) {
- throw new IllegalStateException("Expected " + numDocs + " docs, but got " + totalDocs);
- }
- CodecUtil.writeFooter(docsOut);
- CodecUtil.writeFooter(filePointersOut);
- IOUtils.close(docsOut, filePointersOut);
-
- try (IndexOutput dataOut =
- EndiannessReverserUtil.createOutput(
- dir, IndexFileNames.segmentFileName(name, suffix, extension), ioContext)) {
- CodecUtil.writeIndexHeader(dataOut, codecName + "Idx", VERSION_CURRENT, id, suffix);
-
- metaOut.writeInt(numDocs);
- metaOut.writeInt(blockShift);
- metaOut.writeInt(totalChunks + 1);
- metaOut.writeLong(dataOut.getFilePointer());
-
- try (ChecksumIndexInput docsIn =
- EndiannessReverserUtil.openChecksumInput(dir, docsOut.getName(), IOContext.READONCE)) {
- CodecUtil.checkHeader(docsIn, codecName + "Docs", VERSION_CURRENT, VERSION_CURRENT);
- Throwable priorE = null;
- try {
- final LegacyDirectMonotonicWriter docs =
- LegacyDirectMonotonicWriter.getInstance(
- metaOut, dataOut, totalChunks + 1, blockShift);
- long doc = 0;
- docs.add(doc);
- for (int i = 0; i < totalChunks; ++i) {
- doc += docsIn.readVInt();
- docs.add(doc);
- }
- docs.finish();
- if (doc != totalDocs) {
- throw new CorruptIndexException("Docs don't add up", docsIn);
- }
- } catch (Throwable e) {
- priorE = e;
- } finally {
- CodecUtil.checkFooter(docsIn, priorE);
- }
- }
- dir.deleteFile(docsOut.getName());
- docsOut = null;
-
- metaOut.writeLong(dataOut.getFilePointer());
- try (ChecksumIndexInput filePointersIn =
- EndiannessReverserUtil.openChecksumInput(
- dir, filePointersOut.getName(), IOContext.READONCE)) {
- CodecUtil.checkHeader(
- filePointersIn, codecName + "FilePointers", VERSION_CURRENT, VERSION_CURRENT);
- Throwable priorE = null;
- try {
- final LegacyDirectMonotonicWriter filePointers =
- LegacyDirectMonotonicWriter.getInstance(
- metaOut, dataOut, totalChunks + 1, blockShift);
- long fp = 0;
- for (int i = 0; i < totalChunks; ++i) {
- fp += filePointersIn.readVLong();
- filePointers.add(fp);
- }
- if (maxPointer < fp) {
- throw new CorruptIndexException("File pointers don't add up", filePointersIn);
- }
- filePointers.add(maxPointer);
- filePointers.finish();
- } catch (Throwable e) {
- priorE = e;
- } finally {
- CodecUtil.checkFooter(filePointersIn, priorE);
- }
- }
- dir.deleteFile(filePointersOut.getName());
- filePointersOut = null;
-
- metaOut.writeLong(dataOut.getFilePointer());
- metaOut.writeLong(maxPointer);
-
- CodecUtil.writeFooter(dataOut);
- }
- }
-
- @Override
- public void close() throws IOException {
- try {
- IOUtils.close(docsOut, filePointersOut);
- } finally {
- List<String> fileNames = new ArrayList<>();
- if (docsOut != null) {
- fileNames.add(docsOut.getName());
- }
- if (filePointersOut != null) {
- fileNames.add(filePointersOut.getName());
- }
- try {
- IOUtils.deleteFiles(dir, fileNames);
- } finally {
- docsOut = filePointersOut = null;
- }
- }
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingStoredFieldsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingStoredFieldsWriter.java
deleted file mode 100644
index 0c2c7c2b6fa..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingStoredFieldsWriter.java
+++ /dev/null
@@ -1,507 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50.compressing;
-
-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.BYTE_ARR;
-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.FIELDS_EXTENSION;
-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.INDEX_CODEC_NAME;
-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.INDEX_EXTENSION;
-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.META_EXTENSION;
-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.NUMERIC_DOUBLE;
-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.NUMERIC_FLOAT;
-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.NUMERIC_INT;
-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.NUMERIC_LONG;
-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.STRING;
-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.TYPE_BITS;
-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingStoredFieldsReader.VERSION_CURRENT;
-
-import java.io.IOException;
-import org.apache.lucene.backward_codecs.compressing.CompressionMode;
-import org.apache.lucene.backward_codecs.compressing.Compressor;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.StoredFieldsWriter;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.store.ByteBuffersDataOutput;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BitUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.packed.PackedInts;
-
-/**
- * {@link StoredFieldsWriter} impl for {@link Lucene50CompressingStoredFieldsFormat}.
- *
- * @lucene.experimental
- */
-public final class Lucene50CompressingStoredFieldsWriter extends StoredFieldsWriter {
-
- private final String segment;
- private FieldsIndexWriter indexWriter;
- private IndexOutput metaStream, fieldsStream;
-
- private Compressor compressor;
- private final int chunkSize;
- private final int maxDocsPerChunk;
-
- private final ByteBuffersDataOutput bufferedDocs;
- private int[] numStoredFields; // number of stored fields
- private int[] endOffsets; // end offsets in bufferedDocs
- private int docBase; // doc ID at the beginning of the chunk
- private int numBufferedDocs; // docBase + numBufferedDocs == current doc ID
-
- private long numChunks; // number of written chunks
- private long numDirtyChunks; // number of incomplete compressed blocks written
- private long numDirtyDocs; // cumulative number of missing docs in incomplete chunks
-
- /** Sole constructor. */
- Lucene50CompressingStoredFieldsWriter(
- Directory directory,
- SegmentInfo si,
- String segmentSuffix,
- IOContext context,
- String formatName,
- CompressionMode compressionMode,
- int chunkSize,
- int maxDocsPerChunk,
- int blockShift)
- throws IOException {
- assert directory != null;
- this.segment = si.name;
- this.compressor = compressionMode.newCompressor();
- this.chunkSize = chunkSize;
- this.maxDocsPerChunk = maxDocsPerChunk;
- this.docBase = 0;
- this.bufferedDocs = ByteBuffersDataOutput.newResettableInstance();
- this.numStoredFields = new int[16];
- this.endOffsets = new int[16];
- this.numBufferedDocs = 0;
-
- boolean success = false;
- try {
- metaStream =
- EndiannessReverserUtil.createOutput(
- directory,
- IndexFileNames.segmentFileName(segment, segmentSuffix, META_EXTENSION),
- context);
- CodecUtil.writeIndexHeader(
- metaStream, INDEX_CODEC_NAME + "Meta", VERSION_CURRENT, si.getId(), segmentSuffix);
- assert CodecUtil.indexHeaderLength(INDEX_CODEC_NAME + "Meta", segmentSuffix)
- == metaStream.getFilePointer();
-
- fieldsStream =
- EndiannessReverserUtil.createOutput(
- directory,
- IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION),
- context);
- CodecUtil.writeIndexHeader(
- fieldsStream, formatName, VERSION_CURRENT, si.getId(), segmentSuffix);
- assert CodecUtil.indexHeaderLength(formatName, segmentSuffix)
- == fieldsStream.getFilePointer();
-
- indexWriter =
- new FieldsIndexWriter(
- directory,
- segment,
- segmentSuffix,
- INDEX_EXTENSION,
- INDEX_CODEC_NAME,
- si.getId(),
- blockShift,
- context);
-
- metaStream.writeVInt(chunkSize);
- metaStream.writeVInt(PackedInts.VERSION_CURRENT);
-
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(metaStream, fieldsStream, indexWriter);
- }
- }
- }
-
- @Override
- public void close() throws IOException {
- try {
- IOUtils.close(metaStream, fieldsStream, indexWriter, compressor);
- } finally {
- metaStream = null;
- fieldsStream = null;
- indexWriter = null;
- compressor = null;
- }
- }
-
- private int numStoredFieldsInDoc;
-
- @Override
- public void startDocument() throws IOException {}
-
- @Override
- public void finishDocument() throws IOException {
- if (numBufferedDocs == this.numStoredFields.length) {
- final int newLength = ArrayUtil.oversize(numBufferedDocs + 1, 4);
- this.numStoredFields = ArrayUtil.growExact(this.numStoredFields, newLength);
- endOffsets = ArrayUtil.growExact(endOffsets, newLength);
- }
- this.numStoredFields[numBufferedDocs] = numStoredFieldsInDoc;
- numStoredFieldsInDoc = 0;
- endOffsets[numBufferedDocs] = Math.toIntExact(bufferedDocs.size());
- ++numBufferedDocs;
- if (triggerFlush()) {
- flush(false);
- }
- }
-
- private static void saveInts(int[] values, int length, DataOutput out) throws IOException {
- assert length > 0;
- if (length == 1) {
- out.writeVInt(values[0]);
- } else {
- boolean allEqual = true;
- for (int i = 1; i < length; ++i) {
- if (values[i] != values[0]) {
- allEqual = false;
- break;
- }
- }
- if (allEqual) {
- out.writeVInt(0);
- out.writeVInt(values[0]);
- } else {
- long max = 0;
- for (int i = 0; i < length; ++i) {
- max |= values[i];
- }
- final int bitsRequired = PackedInts.bitsRequired(max);
- out.writeVInt(bitsRequired);
- final PackedInts.Writer w =
- PackedInts.getWriterNoHeader(out, PackedInts.Format.PACKED, length, bitsRequired, 1);
- for (int i = 0; i < length; ++i) {
- w.add(values[i]);
- }
- w.finish();
- }
- }
- }
-
- private void writeHeader(
- int docBase,
- int numBufferedDocs,
- int[] numStoredFields,
- int[] lengths,
- boolean sliced,
- boolean dirtyChunk)
- throws IOException {
- final int slicedBit = sliced ? 1 : 0;
- final int dirtyBit = dirtyChunk ? 2 : 0;
-
- // save docBase and numBufferedDocs
- fieldsStream.writeVInt(docBase);
- fieldsStream.writeVInt((numBufferedDocs << 2) | dirtyBit | slicedBit);
-
- // save numStoredFields
- saveInts(numStoredFields, numBufferedDocs, fieldsStream);
-
- // save lengths
- saveInts(lengths, numBufferedDocs, fieldsStream);
- }
-
- private boolean triggerFlush() {
- return bufferedDocs.size() >= chunkSize
- || // chunks of at least chunkSize bytes
- numBufferedDocs >= maxDocsPerChunk;
- }
-
- private void flush(boolean force) throws IOException {
- assert triggerFlush() != force;
- numChunks++;
- if (force) {
- numDirtyChunks++; // incomplete: we had to force this flush
- numDirtyDocs += numBufferedDocs;
- }
- indexWriter.writeIndex(numBufferedDocs, fieldsStream.getFilePointer());
-
- // transform end offsets into lengths
- final int[] lengths = endOffsets;
- for (int i = numBufferedDocs - 1; i > 0; --i) {
- lengths[i] = endOffsets[i] - endOffsets[i - 1];
- assert lengths[i] >= 0;
- }
- final boolean sliced = bufferedDocs.size() >= 2 * chunkSize;
- writeHeader(docBase, numBufferedDocs, numStoredFields, lengths, sliced, force);
-
- // compress stored fields to fieldsStream.
- //
- // TODO: do we need to slice it since we already have the slices in the buffer? Perhaps
- // we should use max-block-bits restriction on the buffer itself, then we won't have to check it
- // here.
- byte[] content = bufferedDocs.toArrayCopy();
- bufferedDocs.reset();
-
- if (sliced) {
- // big chunk, slice it
- for (int compressed = 0; compressed < content.length; compressed += chunkSize) {
- compressor.compress(
- content, compressed, Math.min(chunkSize, content.length - compressed), fieldsStream);
- }
- } else {
- compressor.compress(content, 0, content.length, fieldsStream);
- }
-
- // reset
- docBase += numBufferedDocs;
- numBufferedDocs = 0;
- bufferedDocs.reset();
- }
-
- @Override
- public void writeField(FieldInfo info, int value) throws IOException {
- ++numStoredFieldsInDoc;
- final long infoAndBits = (((long) info.number) << TYPE_BITS) | NUMERIC_INT;
- bufferedDocs.writeVLong(infoAndBits);
- bufferedDocs.writeZInt(value);
- }
-
- @Override
- public void writeField(FieldInfo info, long value) throws IOException {
- ++numStoredFieldsInDoc;
- final long infoAndBits = (((long) info.number) << TYPE_BITS) | NUMERIC_LONG;
- bufferedDocs.writeVLong(infoAndBits);
- writeTLong(EndiannessReverserUtil.wrapDataOutput(bufferedDocs), value);
- }
-
- @Override
- public void writeField(FieldInfo info, float value) throws IOException {
- ++numStoredFieldsInDoc;
- final long infoAndBits = (((long) info.number) << TYPE_BITS) | NUMERIC_FLOAT;
- bufferedDocs.writeVLong(infoAndBits);
- writeZFloat(EndiannessReverserUtil.wrapDataOutput(bufferedDocs), value);
- }
-
- @Override
- public void writeField(FieldInfo info, double value) throws IOException {
- ++numStoredFieldsInDoc;
- final long infoAndBits = (((long) info.number) << TYPE_BITS) | NUMERIC_DOUBLE;
- bufferedDocs.writeVLong(infoAndBits);
- writeZDouble(EndiannessReverserUtil.wrapDataOutput(bufferedDocs), value);
- }
-
- @Override
- public void writeField(FieldInfo info, BytesRef value) throws IOException {
- ++numStoredFieldsInDoc;
- final long infoAndBits = (((long) info.number) << TYPE_BITS) | BYTE_ARR;
- bufferedDocs.writeVLong(infoAndBits);
- bufferedDocs.writeVInt(value.length);
- bufferedDocs.writeBytes(value.bytes, value.offset, value.length);
- }
-
- @Override
- public void writeField(FieldInfo info, String value) throws IOException {
- ++numStoredFieldsInDoc;
- final long infoAndBits = (((long) info.number) << TYPE_BITS) | STRING;
- bufferedDocs.writeVLong(infoAndBits);
- bufferedDocs.writeString(value);
- }
-
- // -0 isn't compressed.
- static final int NEGATIVE_ZERO_FLOAT = Float.floatToIntBits(-0f);
- static final long NEGATIVE_ZERO_DOUBLE = Double.doubleToLongBits(-0d);
-
- // for compression of timestamps
- static final long SECOND = 1000L;
- static final long HOUR = 60 * 60 * SECOND;
- static final long DAY = 24 * HOUR;
- static final int SECOND_ENCODING = 0x40;
- static final int HOUR_ENCODING = 0x80;
- static final int DAY_ENCODING = 0xC0;
-
- /**
- * Writes a float in a variable-length format. Writes between one and five bytes. Small integral
- * values typically take fewer bytes.
- *
- * <p>ZFloat --&gt; Header, Bytes*?
- *
- * <ul>
- * <li>Header --&gt; {@link DataOutput#writeByte Uint8}. When it is equal to 0xFF then the value
- * is negative and stored in the next 4 bytes. Otherwise if the first bit is set then the
- * other bits in the header encode the value plus one and no other bytes are read.
- * Otherwise, the value is a positive float value whose first byte is the header, and 3
- * bytes need to be read to complete it.
- * <li>Bytes --&gt; Potential additional bytes to read depending on the header.
- * </ul>
- */
- static void writeZFloat(DataOutput out, float f) throws IOException {
- int intVal = (int) f;
- final int floatBits = Float.floatToIntBits(f);
-
- if (f == intVal && intVal >= -1 && intVal <= 0x7D && floatBits != NEGATIVE_ZERO_FLOAT) {
- // small integer value [-1..125]: single byte
- out.writeByte((byte) (0x80 | (1 + intVal)));
- } else if ((floatBits >>> 31) == 0) {
- // other positive floats: 4 bytes
- out.writeInt(floatBits);
- } else {
- // other negative float: 5 bytes
- out.writeByte((byte) 0xFF);
- out.writeInt(floatBits);
- }
- }
-
- /**
- * Writes a float in a variable-length format. Writes between one and five bytes. Small integral
- * values typically take fewer bytes.
- *
- * <p>ZFloat --&gt; Header, Bytes*?
- *
- * <ul>
- * <li>Header --&gt; {@link DataOutput#writeByte Uint8}. When it is equal to 0xFF then the value
- * is negative and stored in the next 8 bytes. When it is equal to 0xFE then the value is
- * stored as a float in the next 4 bytes. Otherwise if the first bit is set then the other
- * bits in the header encode the value plus one and no other bytes are read. Otherwise, the
- * value is a positive float value whose first byte is the header, and 7 bytes need to be
- * read to complete it.
- * <li>Bytes --&gt; Potential additional bytes to read depending on the header.
- * </ul>
- */
- static void writeZDouble(DataOutput out, double d) throws IOException {
- int intVal = (int) d;
- final long doubleBits = Double.doubleToLongBits(d);
-
- if (d == intVal && intVal >= -1 && intVal <= 0x7C && doubleBits != NEGATIVE_ZERO_DOUBLE) {
- // small integer value [-1..124]: single byte
- out.writeByte((byte) (0x80 | (intVal + 1)));
- return;
- } else if (d == (float) d) {
- // d has an accurate float representation: 5 bytes
- out.writeByte((byte) 0xFE);
- out.writeInt(Float.floatToIntBits((float) d));
- } else if ((doubleBits >>> 63) == 0) {
- // other positive doubles: 8 bytes
- out.writeLong(doubleBits);
- } else {
- // other negative doubles: 9 bytes
- out.writeByte((byte) 0xFF);
- out.writeLong(doubleBits);
- }
- }
-
- /**
- * Writes a long in a variable-length format. Writes between one and ten bytes. Small values or
- * values representing timestamps with day, hour or second precision typically require fewer
- * bytes.
- *
- * <p>ZLong --&gt; Header, Bytes*?
- *
- * <ul>
- * <li>Header --&gt; The first two bits indicate the compression scheme:
- * <ul>
- * <li>00 - uncompressed
- * <li>01 - multiple of 1000 (second)
- * <li>10 - multiple of 3600000 (hour)
- * <li>11 - multiple of 86400000 (day)
- * </ul>
- * Then the next bit is a continuation bit, indicating whether more bytes need to be read,
- * and the last 5 bits are the lower bits of the encoded value. In order to reconstruct the
- * value, you need to combine the 5 lower bits of the header with a vLong in the next bytes
- * (if the continuation bit is set to 1). Then {@link BitUtil#zigZagDecode(int)
- * zigzag-decode} it and finally multiply by the multiple corresponding to the compression
- * scheme.
- * <li>Bytes --&gt; Potential additional bytes to read depending on the header.
- * </ul>
- */
- // T for "timestamp"
- static void writeTLong(DataOutput out, long l) throws IOException {
- int header;
- if (l % SECOND != 0) {
- header = 0;
- } else if (l % DAY == 0) {
- // timestamp with day precision
- header = DAY_ENCODING;
- l /= DAY;
- } else if (l % HOUR == 0) {
- // timestamp with hour precision, or day precision with a timezone
- header = HOUR_ENCODING;
- l /= HOUR;
- } else {
- // timestamp with second precision
- header = SECOND_ENCODING;
- l /= SECOND;
- }
-
- final long zigZagL = BitUtil.zigZagEncode(l);
- header |= (zigZagL & 0x1F); // last 5 bits
- final long upperBits = zigZagL >>> 5;
- if (upperBits != 0) {
- header |= 0x20;
- }
- out.writeByte((byte) header);
- if (upperBits != 0) {
- out.writeVLong(upperBits);
- }
- }
-
- @Override
- public void finish(int numDocs) throws IOException {
- if (numBufferedDocs > 0) {
- flush(true);
- } else {
- assert bufferedDocs.size() == 0;
- }
- if (docBase != numDocs) {
- throw new RuntimeException(
- "Wrote " + docBase + " docs, finish called with numDocs=" + numDocs);
- }
- indexWriter.finish(numDocs, fieldsStream.getFilePointer(), metaStream);
- metaStream.writeVLong(numChunks);
- metaStream.writeVLong(numDirtyChunks);
- metaStream.writeVLong(numDirtyDocs);
- CodecUtil.writeFooter(metaStream);
- CodecUtil.writeFooter(fieldsStream);
- assert bufferedDocs.size() == 0;
- }
-
- // bulk merge is scary: its caused corruption bugs in the past.
- // we try to be extra safe with this impl, but add an escape hatch to
- // have a workaround for undiscovered bugs.
- static final String BULK_MERGE_ENABLED_SYSPROP =
- Lucene50CompressingStoredFieldsWriter.class.getName() + ".enableBulkMerge";
- static final boolean BULK_MERGE_ENABLED;
-
- static {
- boolean v = true;
- try {
- v = Boolean.parseBoolean(System.getProperty(BULK_MERGE_ENABLED_SYSPROP, "true"));
- } catch (SecurityException _) {
- }
- BULK_MERGE_ENABLED = v;
- }
-
- @Override
- public long ramBytesUsed() {
- return bufferedDocs.ramBytesUsed()
- + numStoredFields.length * Integer.BYTES
- + endOffsets.length * Integer.BYTES;
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsWriter.java
deleted file mode 100644
index 9a5e89695c3..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsWriter.java
+++ /dev/null
@@ -1,809 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50.compressing;
-
-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.FLAGS_BITS;
-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.OFFSETS;
-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.PACKED_BLOCK_SIZE;
-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.PAYLOADS;
-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.POSITIONS;
-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.VECTORS_EXTENSION;
-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.VECTORS_INDEX_CODEC_NAME;
-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.VECTORS_INDEX_EXTENSION;
-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.VECTORS_META_EXTENSION;
-import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.VERSION_CURRENT;
-
-import java.io.IOException;
-import java.util.ArrayDeque;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Deque;
-import java.util.Iterator;
-import java.util.List;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import org.apache.lucene.backward_codecs.compressing.CompressionMode;
-import org.apache.lucene.backward_codecs.compressing.Compressor;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.TermVectorsWriter;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.store.ByteBuffersDataOutput;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.Accountable;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.StringHelper;
-import org.apache.lucene.util.packed.BlockPackedWriter;
-import org.apache.lucene.util.packed.PackedInts;
-
-/**
- * {@link TermVectorsWriter} for {@link Lucene50CompressingTermVectorsFormat}.
- *
- * @lucene.experimental
- */
-public final class Lucene50CompressingTermVectorsWriter extends TermVectorsWriter {
-
- private final String segment;
- private FieldsIndexWriter indexWriter;
- private IndexOutput metaStream, vectorsStream;
-
- private final Compressor compressor;
- private final int chunkSize;
-
- private long numChunks; // number of chunks
- private long numDirtyChunks; // number of incomplete compressed blocks written
- private long numDirtyDocs; // cumulative number of docs in incomplete chunks
-
- /** a pending doc */
- private class DocData {
- final int numFields;
- final Deque<FieldData> fields;
- final int posStart, offStart, payStart;
-
- DocData(int numFields, int posStart, int offStart, int payStart) {
- this.numFields = numFields;
- this.fields = new ArrayDeque<>(numFields);
- this.posStart = posStart;
- this.offStart = offStart;
- this.payStart = payStart;
- }
-
- FieldData addField(
- int fieldNum, int numTerms, boolean positions, boolean offsets, boolean payloads) {
- final FieldData field;
- if (fields.isEmpty()) {
- field =
- new FieldData(
- fieldNum, numTerms, positions, offsets, payloads, posStart, offStart, payStart);
- } else {
- final FieldData last = fields.getLast();
- final int posStart = last.posStart + (last.hasPositions ? last.totalPositions : 0);
- final int offStart = last.offStart + (last.hasOffsets ? last.totalPositions : 0);
- final int payStart = last.payStart + (last.hasPayloads ? last.totalPositions : 0);
- field =
- new FieldData(
- fieldNum, numTerms, positions, offsets, payloads, posStart, offStart, payStart);
- }
- fields.add(field);
- return field;
- }
- }
-
- private DocData addDocData(int numVectorFields) {
- FieldData last = null;
- for (Iterator<DocData> it = pendingDocs.descendingIterator(); it.hasNext(); ) {
- final DocData doc = it.next();
- if (!doc.fields.isEmpty()) {
- last = doc.fields.getLast();
- break;
- }
- }
- final DocData doc;
- if (last == null) {
- doc = new DocData(numVectorFields, 0, 0, 0);
- } else {
- final int posStart = last.posStart + (last.hasPositions ? last.totalPositions : 0);
- final int offStart = last.offStart + (last.hasOffsets ? last.totalPositions : 0);
- final int payStart = last.payStart + (last.hasPayloads ? last.totalPositions : 0);
- doc = new DocData(numVectorFields, posStart, offStart, payStart);
- }
- pendingDocs.add(doc);
- return doc;
- }
-
- /** a pending field */
- private class FieldData {
- final boolean hasPositions, hasOffsets, hasPayloads;
- final int fieldNum, flags, numTerms;
- final int[] freqs, prefixLengths, suffixLengths;
- final int posStart, offStart, payStart;
- int totalPositions;
- int ord;
-
- FieldData(
- int fieldNum,
- int numTerms,
- boolean positions,
- boolean offsets,
- boolean payloads,
- int posStart,
- int offStart,
- int payStart) {
- this.fieldNum = fieldNum;
- this.numTerms = numTerms;
- this.hasPositions = positions;
- this.hasOffsets = offsets;
- this.hasPayloads = payloads;
- this.flags =
- (positions ? POSITIONS : 0) | (offsets ? OFFSETS : 0) | (payloads ? PAYLOADS : 0);
- this.freqs = new int[numTerms];
- this.prefixLengths = new int[numTerms];
- this.suffixLengths = new int[numTerms];
- this.posStart = posStart;
- this.offStart = offStart;
- this.payStart = payStart;
- totalPositions = 0;
- ord = 0;
- }
-
- void addTerm(int freq, int prefixLength, int suffixLength) {
- freqs[ord] = freq;
- prefixLengths[ord] = prefixLength;
- suffixLengths[ord] = suffixLength;
- ++ord;
- }
-
- void addPosition(int position, int startOffset, int length, int payloadLength) {
- if (hasPositions) {
- if (posStart + totalPositions == positionsBuf.length) {
- positionsBuf = ArrayUtil.grow(positionsBuf);
- }
- positionsBuf[posStart + totalPositions] = position;
- }
- if (hasOffsets) {
- if (offStart + totalPositions == startOffsetsBuf.length) {
- final int newLength = ArrayUtil.oversize(offStart + totalPositions, 4);
- startOffsetsBuf = ArrayUtil.growExact(startOffsetsBuf, newLength);
- lengthsBuf = ArrayUtil.growExact(lengthsBuf, newLength);
- }
- startOffsetsBuf[offStart + totalPositions] = startOffset;
- lengthsBuf[offStart + totalPositions] = length;
- }
- if (hasPayloads) {
- if (payStart + totalPositions == payloadLengthsBuf.length) {
- payloadLengthsBuf = ArrayUtil.grow(payloadLengthsBuf);
- }
- payloadLengthsBuf[payStart + totalPositions] = payloadLength;
- }
- ++totalPositions;
- }
- }
-
- private int numDocs; // total number of docs seen
- private final Deque<DocData> pendingDocs; // pending docs
- private DocData curDoc; // current document
- private FieldData curField; // current field
- private final BytesRef lastTerm;
- private int[] positionsBuf, startOffsetsBuf, lengthsBuf, payloadLengthsBuf;
- private final ByteBuffersDataOutput termSuffixes; // buffered term suffixes
- private final ByteBuffersDataOutput payloadBytes; // buffered term payloads
- private final BlockPackedWriter writer;
- private final int maxDocsPerChunk; // hard limit on number of docs per chunk
-
- /** Sole constructor. */
- Lucene50CompressingTermVectorsWriter(
- Directory directory,
- SegmentInfo si,
- String segmentSuffix,
- IOContext context,
- String formatName,
- CompressionMode compressionMode,
- int chunkSize,
- int maxDocsPerChunk,
- int blockShift)
- throws IOException {
- assert directory != null;
- this.segment = si.name;
- this.compressor = compressionMode.newCompressor();
- this.chunkSize = chunkSize;
- this.maxDocsPerChunk = maxDocsPerChunk;
-
- numDocs = 0;
- pendingDocs = new ArrayDeque<>();
- termSuffixes = ByteBuffersDataOutput.newResettableInstance();
- payloadBytes = ByteBuffersDataOutput.newResettableInstance();
- lastTerm = new BytesRef(ArrayUtil.oversize(30, 1));
-
- boolean success = false;
- try {
- metaStream =
- EndiannessReverserUtil.createOutput(
- directory,
- IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_META_EXTENSION),
- context);
- CodecUtil.writeIndexHeader(
- metaStream,
- VECTORS_INDEX_CODEC_NAME + "Meta",
- VERSION_CURRENT,
- si.getId(),
- segmentSuffix);
- assert CodecUtil.indexHeaderLength(VECTORS_INDEX_CODEC_NAME + "Meta", segmentSuffix)
- == metaStream.getFilePointer();
-
- vectorsStream =
- EndiannessReverserUtil.createOutput(
- directory,
- IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION),
- context);
- CodecUtil.writeIndexHeader(
- vectorsStream, formatName, VERSION_CURRENT, si.getId(), segmentSuffix);
- assert CodecUtil.indexHeaderLength(formatName, segmentSuffix)
- == vectorsStream.getFilePointer();
-
- indexWriter =
- new FieldsIndexWriter(
- directory,
- segment,
- segmentSuffix,
- VECTORS_INDEX_EXTENSION,
- VECTORS_INDEX_CODEC_NAME,
- si.getId(),
- blockShift,
- context);
-
- metaStream.writeVInt(PackedInts.VERSION_CURRENT);
- metaStream.writeVInt(chunkSize);
- writer = new BlockPackedWriter(vectorsStream, PACKED_BLOCK_SIZE);
-
- positionsBuf = new int[1024];
- startOffsetsBuf = new int[1024];
- lengthsBuf = new int[1024];
- payloadLengthsBuf = new int[1024];
-
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(metaStream, vectorsStream, indexWriter, indexWriter);
- }
- }
- }
-
- @Override
- public void close() throws IOException {
- try {
- IOUtils.close(metaStream, vectorsStream, indexWriter);
- } finally {
- metaStream = null;
- vectorsStream = null;
- indexWriter = null;
- }
- }
-
- @Override
- public void startDocument(int numVectorFields) throws IOException {
- curDoc = addDocData(numVectorFields);
- }
-
- @Override
- public void finishDocument() throws IOException {
- // append the payload bytes of the doc after its terms
- payloadBytes.copyTo(termSuffixes);
- payloadBytes.reset();
- ++numDocs;
- if (triggerFlush()) {
- flush();
- }
- curDoc = null;
- }
-
- @Override
- public void startField(
- FieldInfo info, int numTerms, boolean positions, boolean offsets, boolean payloads)
- throws IOException {
- curField = curDoc.addField(info.number, numTerms, positions, offsets, payloads);
- lastTerm.length = 0;
- }
-
- @Override
- public void finishField() throws IOException {
- curField = null;
- }
-
- @Override
- public void startTerm(BytesRef term, int freq) throws IOException {
- assert freq >= 1;
- final int prefix;
- if (lastTerm.length == 0) {
- // no previous term: no bytes to write
- prefix = 0;
- } else {
- prefix = StringHelper.bytesDifference(lastTerm, term);
- }
- curField.addTerm(freq, prefix, term.length - prefix);
- termSuffixes.writeBytes(term.bytes, term.offset + prefix, term.length - prefix);
- // copy last term
- if (lastTerm.bytes.length < term.length) {
- lastTerm.bytes = new byte[ArrayUtil.oversize(term.length, 1)];
- }
- lastTerm.offset = 0;
- lastTerm.length = term.length;
- System.arraycopy(term.bytes, term.offset, lastTerm.bytes, 0, term.length);
- }
-
- @Override
- public void addPosition(int position, int startOffset, int endOffset, BytesRef payload)
- throws IOException {
- assert curField.flags != 0;
- curField.addPosition(
- position, startOffset, endOffset - startOffset, payload == null ? 0 : payload.length);
- if (curField.hasPayloads && payload != null) {
- payloadBytes.writeBytes(payload.bytes, payload.offset, payload.length);
- }
- }
-
- private boolean triggerFlush() {
- return termSuffixes.size() >= chunkSize || pendingDocs.size() >= maxDocsPerChunk;
- }
-
- private void flush() throws IOException {
- numChunks++;
- final int chunkDocs = pendingDocs.size();
- assert chunkDocs > 0 : chunkDocs;
-
- // write the index file
- indexWriter.writeIndex(chunkDocs, vectorsStream.getFilePointer());
-
- final int docBase = numDocs - chunkDocs;
- vectorsStream.writeVInt(docBase);
- vectorsStream.writeVInt(chunkDocs);
-
- // total number of fields of the chunk
- final int totalFields = flushNumFields(chunkDocs);
-
- if (totalFields > 0) {
- // unique field numbers (sorted)
- final int[] fieldNums = flushFieldNums();
- // offsets in the array of unique field numbers
- flushFields(totalFields, fieldNums);
- // flags (does the field have positions, offsets, payloads?)
- flushFlags(totalFields, fieldNums);
- // number of terms of each field
- flushNumTerms(totalFields);
- // prefix and suffix lengths for each field
- flushTermLengths();
- // term freqs - 1 (because termFreq is always >=1) for each term
- flushTermFreqs();
- // positions for all terms, when enabled
- flushPositions();
- // offsets for all terms, when enabled
- flushOffsets(fieldNums);
- // payload lengths for all terms, when enabled
- flushPayloadLengths();
-
- // compress terms and payloads and write them to the output
- //
- // TODO: We could compress in the slices we already have in the buffer (min/max slice
- // can be set on the buffer itself).
- byte[] content = termSuffixes.toArrayCopy();
- compressor.compress(content, 0, content.length, vectorsStream);
- }
-
- // reset
- pendingDocs.clear();
- curDoc = null;
- curField = null;
- termSuffixes.reset();
- }
-
- private int flushNumFields(int chunkDocs) throws IOException {
- if (chunkDocs == 1) {
- final int numFields = pendingDocs.getFirst().numFields;
- vectorsStream.writeVInt(numFields);
- return numFields;
- } else {
- writer.reset(vectorsStream);
- int totalFields = 0;
- for (DocData dd : pendingDocs) {
- writer.add(dd.numFields);
- totalFields += dd.numFields;
- }
- writer.finish();
- return totalFields;
- }
- }
-
- /** Returns a sorted array containing unique field numbers */
- private int[] flushFieldNums() throws IOException {
- SortedSet<Integer> fieldNums = new TreeSet<>();
- for (DocData dd : pendingDocs) {
- for (FieldData fd : dd.fields) {
- fieldNums.add(fd.fieldNum);
- }
- }
-
- final int numDistinctFields = fieldNums.size();
- assert numDistinctFields > 0;
- final int bitsRequired = PackedInts.bitsRequired(fieldNums.last());
- final int token = (Math.min(numDistinctFields - 1, 0x07) << 5) | bitsRequired;
- vectorsStream.writeByte((byte) token);
- if (numDistinctFields - 1 >= 0x07) {
- vectorsStream.writeVInt(numDistinctFields - 1 - 0x07);
- }
- final PackedInts.Writer writer =
- PackedInts.getWriterNoHeader(
- vectorsStream, PackedInts.Format.PACKED, fieldNums.size(), bitsRequired, 1);
- for (Integer fieldNum : fieldNums) {
- writer.add(fieldNum);
- }
- writer.finish();
-
- int[] fns = new int[fieldNums.size()];
- int i = 0;
- for (Integer key : fieldNums) {
- fns[i++] = key;
- }
- return fns;
- }
-
- private void flushFields(int totalFields, int[] fieldNums) throws IOException {
- final PackedInts.Writer writer =
- PackedInts.getWriterNoHeader(
- vectorsStream,
- PackedInts.Format.PACKED,
- totalFields,
- PackedInts.bitsRequired(fieldNums.length - 1),
- 1);
- for (DocData dd : pendingDocs) {
- for (FieldData fd : dd.fields) {
- final int fieldNumIndex = Arrays.binarySearch(fieldNums, fd.fieldNum);
- assert fieldNumIndex >= 0;
- writer.add(fieldNumIndex);
- }
- }
- writer.finish();
- }
-
- private void flushFlags(int totalFields, int[] fieldNums) throws IOException {
- // check if fields always have the same flags
- boolean nonChangingFlags = true;
- int[] fieldFlags = new int[fieldNums.length];
- Arrays.fill(fieldFlags, -1);
- outer:
- for (DocData dd : pendingDocs) {
- for (FieldData fd : dd.fields) {
- final int fieldNumOff = Arrays.binarySearch(fieldNums, fd.fieldNum);
- assert fieldNumOff >= 0;
- if (fieldFlags[fieldNumOff] == -1) {
- fieldFlags[fieldNumOff] = fd.flags;
- } else if (fieldFlags[fieldNumOff] != fd.flags) {
- nonChangingFlags = false;
- break outer;
- }
- }
- }
-
- if (nonChangingFlags) {
- // write one flag per field num
- vectorsStream.writeVInt(0);
- final PackedInts.Writer writer =
- PackedInts.getWriterNoHeader(
- vectorsStream, PackedInts.Format.PACKED, fieldFlags.length, FLAGS_BITS, 1);
- for (int flags : fieldFlags) {
- assert flags >= 0;
- writer.add(flags);
- }
- assert writer.ord() == fieldFlags.length - 1;
- writer.finish();
- } else {
- // write one flag for every field instance
- vectorsStream.writeVInt(1);
- final PackedInts.Writer writer =
- PackedInts.getWriterNoHeader(
- vectorsStream, PackedInts.Format.PACKED, totalFields, FLAGS_BITS, 1);
- for (DocData dd : pendingDocs) {
- for (FieldData fd : dd.fields) {
- writer.add(fd.flags);
- }
- }
- assert writer.ord() == totalFields - 1;
- writer.finish();
- }
- }
-
- private void flushNumTerms(int totalFields) throws IOException {
- int maxNumTerms = 0;
- for (DocData dd : pendingDocs) {
- for (FieldData fd : dd.fields) {
- maxNumTerms |= fd.numTerms;
- }
- }
- final int bitsRequired = PackedInts.bitsRequired(maxNumTerms);
- vectorsStream.writeVInt(bitsRequired);
- final PackedInts.Writer writer =
- PackedInts.getWriterNoHeader(
- vectorsStream, PackedInts.Format.PACKED, totalFields, bitsRequired, 1);
- for (DocData dd : pendingDocs) {
- for (FieldData fd : dd.fields) {
- writer.add(fd.numTerms);
- }
- }
- assert writer.ord() == totalFields - 1;
- writer.finish();
- }
-
- private void flushTermLengths() throws IOException {
- writer.reset(vectorsStream);
- for (DocData dd : pendingDocs) {
- for (FieldData fd : dd.fields) {
- for (int i = 0; i < fd.numTerms; ++i) {
- writer.add(fd.prefixLengths[i]);
- }
- }
- }
- writer.finish();
- writer.reset(vectorsStream);
- for (DocData dd : pendingDocs) {
- for (FieldData fd : dd.fields) {
- for (int i = 0; i < fd.numTerms; ++i) {
- writer.add(fd.suffixLengths[i]);
- }
- }
- }
- writer.finish();
- }
-
- private void flushTermFreqs() throws IOException {
- writer.reset(vectorsStream);
- for (DocData dd : pendingDocs) {
- for (FieldData fd : dd.fields) {
- for (int i = 0; i < fd.numTerms; ++i) {
- writer.add(fd.freqs[i] - 1);
- }
- }
- }
- writer.finish();
- }
-
- private void flushPositions() throws IOException {
- writer.reset(vectorsStream);
- for (DocData dd : pendingDocs) {
- for (FieldData fd : dd.fields) {
- if (fd.hasPositions) {
- int pos = 0;
- for (int i = 0; i < fd.numTerms; ++i) {
- int previousPosition = 0;
- for (int j = 0; j < fd.freqs[i]; ++j) {
- final int position = positionsBuf[fd.posStart + pos++];
- writer.add(position - previousPosition);
- previousPosition = position;
- }
- }
- assert pos == fd.totalPositions;
- }
- }
- }
- writer.finish();
- }
-
- private void flushOffsets(int[] fieldNums) throws IOException {
- boolean hasOffsets = false;
- long[] sumPos = new long[fieldNums.length];
- long[] sumOffsets = new long[fieldNums.length];
- for (DocData dd : pendingDocs) {
- for (FieldData fd : dd.fields) {
- hasOffsets |= fd.hasOffsets;
- if (fd.hasOffsets && fd.hasPositions) {
- final int fieldNumOff = Arrays.binarySearch(fieldNums, fd.fieldNum);
- int pos = 0;
- for (int i = 0; i < fd.numTerms; ++i) {
- sumPos[fieldNumOff] += positionsBuf[fd.posStart + fd.freqs[i] - 1 + pos];
- sumOffsets[fieldNumOff] += startOffsetsBuf[fd.offStart + fd.freqs[i] - 1 + pos];
- pos += fd.freqs[i];
- }
- assert pos == fd.totalPositions;
- }
- }
- }
-
- if (!hasOffsets) {
- // nothing to do
- return;
- }
-
- final float[] charsPerTerm = new float[fieldNums.length];
- for (int i = 0; i < fieldNums.length; ++i) {
- charsPerTerm[i] =
- (sumPos[i] <= 0 || sumOffsets[i] <= 0) ? 0 : (float) ((double) sumOffsets[i] / sumPos[i]);
- }
-
- // start offsets
- for (int i = 0; i < fieldNums.length; ++i) {
- vectorsStream.writeInt(Float.floatToRawIntBits(charsPerTerm[i]));
- }
-
- writer.reset(vectorsStream);
- for (DocData dd : pendingDocs) {
- for (FieldData fd : dd.fields) {
- if ((fd.flags & OFFSETS) != 0) {
- final int fieldNumOff = Arrays.binarySearch(fieldNums, fd.fieldNum);
- final float cpt = charsPerTerm[fieldNumOff];
- int pos = 0;
- for (int i = 0; i < fd.numTerms; ++i) {
- int previousPos = 0;
- int previousOff = 0;
- for (int j = 0; j < fd.freqs[i]; ++j) {
- final int position = fd.hasPositions ? positionsBuf[fd.posStart + pos] : 0;
- final int startOffset = startOffsetsBuf[fd.offStart + pos];
- writer.add(startOffset - previousOff - (int) (cpt * (position - previousPos)));
- previousPos = position;
- previousOff = startOffset;
- ++pos;
- }
- }
- }
- }
- }
- writer.finish();
-
- // lengths
- writer.reset(vectorsStream);
- for (DocData dd : pendingDocs) {
- for (FieldData fd : dd.fields) {
- if ((fd.flags & OFFSETS) != 0) {
- int pos = 0;
- for (int i = 0; i < fd.numTerms; ++i) {
- for (int j = 0; j < fd.freqs[i]; ++j) {
- writer.add(
- lengthsBuf[fd.offStart + pos++] - fd.prefixLengths[i] - fd.suffixLengths[i]);
- }
- }
- assert pos == fd.totalPositions;
- }
- }
- }
- writer.finish();
- }
-
- private void flushPayloadLengths() throws IOException {
- writer.reset(vectorsStream);
- for (DocData dd : pendingDocs) {
- for (FieldData fd : dd.fields) {
- if (fd.hasPayloads) {
- for (int i = 0; i < fd.totalPositions; ++i) {
- writer.add(payloadLengthsBuf[fd.payStart + i]);
- }
- }
- }
- }
- writer.finish();
- }
-
- @Override
- public void finish(int numDocs) throws IOException {
- if (!pendingDocs.isEmpty()) {
- numDirtyChunks++; // incomplete: we had to force this flush
- numDirtyDocs += pendingDocs.size();
- flush();
- }
- if (numDocs != this.numDocs) {
- throw new RuntimeException(
- "Wrote " + this.numDocs + " docs, finish called with numDocs=" + numDocs);
- }
- indexWriter.finish(numDocs, vectorsStream.getFilePointer(), metaStream);
- metaStream.writeVLong(numChunks);
- metaStream.writeVLong(numDirtyChunks);
- metaStream.writeVLong(numDirtyDocs);
- CodecUtil.writeFooter(metaStream);
- CodecUtil.writeFooter(vectorsStream);
- }
-
- @Override
- public void addProx(int numProx, DataInput positions, DataInput offsets) throws IOException {
- assert (curField.hasPositions) == (positions != null);
- assert (curField.hasOffsets) == (offsets != null);
-
- if (curField.hasPositions) {
- final int posStart = curField.posStart + curField.totalPositions;
- if (posStart + numProx > positionsBuf.length) {
- positionsBuf = ArrayUtil.grow(positionsBuf, posStart + numProx);
- }
- int position = 0;
- if (curField.hasPayloads) {
- final int payStart = curField.payStart + curField.totalPositions;
- if (payStart + numProx > payloadLengthsBuf.length) {
- payloadLengthsBuf = ArrayUtil.grow(payloadLengthsBuf, payStart + numProx);
- }
- for (int i = 0; i < numProx; ++i) {
- final int code = positions.readVInt();
- if ((code & 1) != 0) {
- // This position has a payload
- final int payloadLength = positions.readVInt();
- payloadLengthsBuf[payStart + i] = payloadLength;
- payloadBytes.copyBytes(positions, payloadLength);
- } else {
- payloadLengthsBuf[payStart + i] = 0;
- }
- position += code >>> 1;
- positionsBuf[posStart + i] = position;
- }
- } else {
- for (int i = 0; i < numProx; ++i) {
- position += (positions.readVInt() >>> 1);
- positionsBuf[posStart + i] = position;
- }
- }
- }
-
- if (curField.hasOffsets) {
- final int offStart = curField.offStart + curField.totalPositions;
- if (offStart + numProx > startOffsetsBuf.length) {
- final int newLength = ArrayUtil.oversize(offStart + numProx, 4);
- startOffsetsBuf = ArrayUtil.growExact(startOffsetsBuf, newLength);
- lengthsBuf = ArrayUtil.growExact(lengthsBuf, newLength);
- }
- int lastOffset = 0, startOffset, endOffset;
- for (int i = 0; i < numProx; ++i) {
- startOffset = lastOffset + offsets.readVInt();
- endOffset = startOffset + offsets.readVInt();
- lastOffset = endOffset;
- startOffsetsBuf[offStart + i] = startOffset;
- lengthsBuf[offStart + i] = endOffset - startOffset;
- }
- }
-
- curField.totalPositions += numProx;
- }
-
- // bulk merge is scary: its caused corruption bugs in the past.
- // we try to be extra safe with this impl, but add an escape hatch to
- // have a workaround for undiscovered bugs.
- static final String BULK_MERGE_ENABLED_SYSPROP =
- Lucene50CompressingTermVectorsWriter.class.getName() + ".enableBulkMerge";
- static final boolean BULK_MERGE_ENABLED;
-
- static {
- boolean v = true;
- try {
- v = Boolean.parseBoolean(System.getProperty(BULK_MERGE_ENABLED_SYSPROP, "true"));
- } catch (SecurityException _) {
- }
- BULK_MERGE_ENABLED = v;
- }
-
- @Override
- public long ramBytesUsed() {
- return positionsBuf.length
- + startOffsetsBuf.length
- + lengthsBuf.length
- + payloadLengthsBuf.length
- + termSuffixes.ramBytesUsed()
- + payloadBytes.ramBytesUsed()
- + lastTerm.bytes.length;
- }
-
- @Override
- public Collection<Accountable> getChildResources() {
- return List.of(termSuffixes, payloadBytes);
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50RWCompressingStoredFieldsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50RWCompressingStoredFieldsFormat.java
deleted file mode 100644
index 6983944ed5e..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50RWCompressingStoredFieldsFormat.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50.compressing;
-
-import java.io.IOException;
-import org.apache.lucene.backward_codecs.compressing.CompressionMode;
-import org.apache.lucene.codecs.StoredFieldsWriter;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-
-/** RW impersonation of Lucene50CompressingStoredFieldsFormat. */
-public class Lucene50RWCompressingStoredFieldsFormat extends Lucene50CompressingStoredFieldsFormat {
-
- public Lucene50RWCompressingStoredFieldsFormat(
- String formatName,
- CompressionMode compressionMode,
- int chunkSize,
- int maxDocsPerChunk,
- int blockShift) {
- super(formatName, compressionMode, chunkSize, maxDocsPerChunk, blockShift);
- }
-
- @Override
- public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context)
- throws IOException {
- return new Lucene50CompressingStoredFieldsWriter(
- directory,
- si,
- segmentSuffix,
- context,
- formatName,
- compressionMode,
- chunkSize,
- maxDocsPerChunk,
- blockShift);
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50RWCompressingTermVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50RWCompressingTermVectorsFormat.java
deleted file mode 100644
index f10b1418413..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50RWCompressingTermVectorsFormat.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene50.compressing;
-
-import java.io.IOException;
-import org.apache.lucene.backward_codecs.compressing.CompressionMode;
-import org.apache.lucene.codecs.TermVectorsWriter;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-
-/** RW impersonation of Lucene50CompressingTermVectorsFormat. */
-public class Lucene50RWCompressingTermVectorsFormat extends Lucene50CompressingTermVectorsFormat {
-
- /** Sole constructor. */
- public Lucene50RWCompressingTermVectorsFormat(
- String formatName,
- String segmentSuffix,
- CompressionMode compressionMode,
- int chunkSize,
- int maxDocsPerChunk,
- int blockSize) {
- super(formatName, segmentSuffix, compressionMode, chunkSize, maxDocsPerChunk, blockSize);
- }
-
- @Override
- public final TermVectorsWriter vectorsWriter(
- Directory directory, SegmentInfo segmentInfo, IOContext context) throws IOException {
- return new Lucene50CompressingTermVectorsWriter(
- directory,
- segmentInfo,
- segmentSuffix,
- context,
- formatName,
- compressionMode,
- chunkSize,
- maxDocsPerChunk,
- blockSize);
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/Lucene60PointsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/Lucene60PointsWriter.java
deleted file mode 100644
index 62f227c699e..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/Lucene60PointsWriter.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene60;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import org.apache.lucene.backward_codecs.lucene60.bkd.BKDWriter60;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.MutablePointTree;
-import org.apache.lucene.codecs.PointsReader;
-import org.apache.lucene.codecs.PointsWriter;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.MergeState;
-import org.apache.lucene.index.PointValues;
-import org.apache.lucene.index.PointValues.IntersectVisitor;
-import org.apache.lucene.index.PointValues.Relation;
-import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.IOUtils;
-
-/** Writes dimensional values */
-public class Lucene60PointsWriter extends PointsWriter {
-
- /** Output used to write the BKD tree data file */
- protected final IndexOutput dataOut;
-
- /** Maps field name to file pointer in the data file where the BKD index is located. */
- protected final Map<String, Long> indexFPs = new HashMap<>();
-
- final SegmentWriteState writeState;
- final int maxPointsInLeafNode;
- final double maxMBSortInHeap;
- private boolean finished;
-
- /** Full constructor */
- public Lucene60PointsWriter(
- SegmentWriteState writeState, int maxPointsInLeafNode, double maxMBSortInHeap)
- throws IOException {
- assert writeState.fieldInfos.hasPointValues();
- this.writeState = writeState;
- this.maxPointsInLeafNode = maxPointsInLeafNode;
- this.maxMBSortInHeap = maxMBSortInHeap;
- String dataFileName =
- IndexFileNames.segmentFileName(
- writeState.segmentInfo.name,
- writeState.segmentSuffix,
- Lucene60PointsFormat.DATA_EXTENSION);
- dataOut =
- EndiannessReverserUtil.createOutput(writeState.directory, dataFileName, writeState.context);
- boolean success = false;
- try {
- CodecUtil.writeIndexHeader(
- dataOut,
- Lucene60PointsFormat.DATA_CODEC_NAME,
- Lucene60PointsFormat.DATA_VERSION_CURRENT,
- writeState.segmentInfo.getId(),
- writeState.segmentSuffix);
- success = true;
- } finally {
- if (success == false) {
- IOUtils.closeWhileHandlingException(dataOut);
- }
- }
- }
-
- /**
- * Uses the defaults values for {@code maxPointsInLeafNode} (1024) and {@code maxMBSortInHeap}
- * (16.0)
- */
- public Lucene60PointsWriter(SegmentWriteState writeState) throws IOException {
- this(
- writeState,
- BKDWriter60.DEFAULT_MAX_POINTS_IN_LEAF_NODE,
- BKDWriter60.DEFAULT_MAX_MB_SORT_IN_HEAP);
- }
-
- @Override
- public void writeField(FieldInfo fieldInfo, PointsReader reader) throws IOException {
-
- PointValues.PointTree values = reader.getValues(fieldInfo.name).getPointTree();
-
- try (BKDWriter60 writer =
- new BKDWriter60(
- writeState.segmentInfo.maxDoc(),
- writeState.directory,
- writeState.segmentInfo.name,
- fieldInfo.getPointDimensionCount(),
- fieldInfo.getPointIndexDimensionCount(),
- fieldInfo.getPointNumBytes(),
- maxPointsInLeafNode,
- maxMBSortInHeap,
- values.size())) {
-
- if (values instanceof MutablePointTree) {
- final long fp = writer.writeField(dataOut, fieldInfo.name, (MutablePointTree) values);
- if (fp != -1) {
- indexFPs.put(fieldInfo.name, fp);
- }
- return;
- }
-
- values.visitDocValues(
- new IntersectVisitor() {
- @Override
- public void visit(int docID) {
- throw new IllegalStateException();
- }
-
- @Override
- public void visit(int docID, byte[] packedValue) throws IOException {
- writer.add(packedValue, docID);
- }
-
- @Override
- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- return Relation.CELL_CROSSES_QUERY;
- }
- });
-
- // We could have 0 points on merge since all docs with dimensional fields may be deleted:
- if (writer.getPointCount() > 0) {
- indexFPs.put(fieldInfo.name, writer.finish(dataOut));
- }
- }
- }
-
- @Override
- public void merge(MergeState mergeState) throws IOException {
- /*
- * If indexSort is activated and some of the leaves are not sorted the next test will catch that
- * and the non-optimized merge will run. If the readers are all sorted then it's safe to perform
- * a bulk merge of the points.
- */
- for (PointsReader reader : mergeState.pointsReaders) {
- if (reader instanceof Lucene60PointsReader == false) {
- // We can only bulk merge when all to-be-merged segments use our format:
- super.merge(mergeState);
- return;
- }
- }
- for (PointsReader reader : mergeState.pointsReaders) {
- if (reader != null) {
- reader.checkIntegrity();
- }
- }
-
- for (FieldInfo fieldInfo : mergeState.mergeFieldInfos) {
- if (fieldInfo.getPointDimensionCount() != 0) {
- if (fieldInfo.getPointDimensionCount() == 1) {
-
- // Worst case total maximum size (if none of the points are deleted):
- long totMaxSize = 0;
- for (int i = 0; i < mergeState.pointsReaders.length; i++) {
- PointsReader reader = mergeState.pointsReaders[i];
- if (reader != null) {
- FieldInfos readerFieldInfos = mergeState.fieldInfos[i];
- FieldInfo readerFieldInfo = readerFieldInfos.fieldInfo(fieldInfo.name);
- if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {
- PointValues values = reader.getValues(fieldInfo.name);
- if (values != null) {
- totMaxSize += values.size();
- }
- }
- }
- }
-
- // System.out.println("MERGE: field=" + fieldInfo.name);
- // Optimize the 1D case to use BKDWriter.merge, which does a single merge sort of the
- // already sorted incoming segments, instead of trying to sort all points again as if
- // we were simply reindexing them:
- try (BKDWriter60 writer =
- new BKDWriter60(
- writeState.segmentInfo.maxDoc(),
- writeState.directory,
- writeState.segmentInfo.name,
- fieldInfo.getPointDimensionCount(),
- fieldInfo.getPointIndexDimensionCount(),
- fieldInfo.getPointNumBytes(),
- maxPointsInLeafNode,
- maxMBSortInHeap,
- totMaxSize)) {
- List<PointValues> bkdReaders = new ArrayList<>();
- List<MergeState.DocMap> docMaps = new ArrayList<>();
- for (int i = 0; i < mergeState.pointsReaders.length; i++) {
- PointsReader reader = mergeState.pointsReaders[i];
-
- if (reader != null) {
-
- // we confirmed this up above
- assert reader instanceof Lucene60PointsReader;
- Lucene60PointsReader reader60 = (Lucene60PointsReader) reader;
-
- // NOTE: we cannot just use the merged fieldInfo.number (instead of resolving to
- // this
- // reader's FieldInfo as we do below) because field numbers can easily be different
- // when addIndexes(Directory...) copies over segments from another index:
-
- FieldInfos readerFieldInfos = mergeState.fieldInfos[i];
- FieldInfo readerFieldInfo = readerFieldInfos.fieldInfo(fieldInfo.name);
- if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {
- PointValues bkdReader = reader60.getValues(readerFieldInfo.name);
- if (bkdReader != null) {
- bkdReaders.add(bkdReader);
- docMaps.add(mergeState.docMaps[i]);
- }
- }
- }
- }
-
- long fp = writer.merge(dataOut, docMaps, bkdReaders);
- if (fp != -1) {
- indexFPs.put(fieldInfo.name, fp);
- }
- }
- } else {
- mergeOneField(mergeState, fieldInfo);
- }
- }
- }
-
- finish();
- }
-
- @Override
- public void finish() throws IOException {
- if (finished) {
- throw new IllegalStateException("already finished");
- }
- finished = true;
- CodecUtil.writeFooter(dataOut);
-
- String indexFileName =
- IndexFileNames.segmentFileName(
- writeState.segmentInfo.name,
- writeState.segmentSuffix,
- Lucene60PointsFormat.INDEX_EXTENSION);
- // Write index file
- try (IndexOutput indexOut =
- EndiannessReverserUtil.createOutput(
- writeState.directory, indexFileName, writeState.context)) {
- CodecUtil.writeIndexHeader(
- indexOut,
- Lucene60PointsFormat.META_CODEC_NAME,
- Lucene60PointsFormat.INDEX_VERSION_CURRENT,
- writeState.segmentInfo.getId(),
- writeState.segmentSuffix);
- int count = indexFPs.size();
- indexOut.writeVInt(count);
- for (Map.Entry<String, Long> ent : indexFPs.entrySet()) {
- FieldInfo fieldInfo = writeState.fieldInfos.fieldInfo(ent.getKey());
- if (fieldInfo == null) {
- throw new IllegalStateException(
- "wrote field=\"" + ent.getKey() + "\" but that field doesn't exist in FieldInfos");
- }
- indexOut.writeVInt(fieldInfo.number);
- indexOut.writeVLong(ent.getValue());
- }
- CodecUtil.writeFooter(indexOut);
- }
- }
-
- @Override
- public void close() throws IOException {
- dataOut.close();
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/Lucene60RWPointsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/Lucene60RWPointsFormat.java
deleted file mode 100644
index 3df9edd7360..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/Lucene60RWPointsFormat.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene60;
-
-import java.io.IOException;
-import org.apache.lucene.codecs.PointsWriter;
-import org.apache.lucene.index.SegmentWriteState;
-
-/** RW variant of Lucene60PointsFormat */
-public class Lucene60RWPointsFormat extends Lucene60PointsFormat {
-
- /** Sole constructor. */
- public Lucene60RWPointsFormat() {}
-
- @Override
- public PointsWriter fieldsWriter(SegmentWriteState state) throws IOException {
- return new Lucene60PointsWriter(state);
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/TestLucene60FieldInfosFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/TestLucene60FieldInfosFormat.java
deleted file mode 100644
index 0b9e9f72672..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/TestLucene60FieldInfosFormat.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene60;
-
-import org.apache.lucene.backward_codecs.lucene84.Lucene84RWCodec;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.tests.index.BaseFieldInfoFormatTestCase;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestLucene60FieldInfosFormat extends BaseFieldInfoFormatTestCase {
- @Override
- protected Codec getCodec() {
- return new Lucene84RWCodec();
- }
-
- @Override
- protected boolean supportDocValuesSkipIndex() {
- return false;
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/TestLucene60PointsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/TestLucene60PointsFormat.java
deleted file mode 100644
index ee1cd9f66bb..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/TestLucene60PointsFormat.java
+++ /dev/null
@@ -1,320 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene60;
-
-import java.io.IOException;
-import java.util.Arrays;
-import org.apache.lucene.backward_codecs.lucene60.bkd.BKDWriter60;
-import org.apache.lucene.backward_codecs.lucene84.Lucene84RWCodec;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.document.BinaryPoint;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.PointValues;
-import org.apache.lucene.index.PointValues.IntersectVisitor;
-import org.apache.lucene.index.PointValues.Relation;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.tests.index.BasePointsFormatTestCase;
-import org.apache.lucene.tests.index.MockRandomMergePolicy;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-import org.apache.lucene.tests.util.TestUtil;
-
-/** Tests Lucene60PointsFormat */
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestLucene60PointsFormat extends BasePointsFormatTestCase {
- private final Codec codec;
- private final int maxPointsInLeafNode;
-
- public TestLucene60PointsFormat() {
- codec = new Lucene84RWCodec();
- maxPointsInLeafNode = BKDWriter60.DEFAULT_MAX_POINTS_IN_LEAF_NODE;
- }
-
- @Override
- protected Codec getCodec() {
- return codec;
- }
-
- // TODO: clean up the math/estimation here rather than suppress so many warnings
- @SuppressWarnings({"NarrowCalculation", "LongDoubleConversion"})
- public void testEstimatePointCount() throws IOException {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig();
- // Avoid mockRandomMP since it may cause non-optimal merges that make the
- // number of points per leaf hard to predict
- while (iwc.getMergePolicy() instanceof MockRandomMergePolicy) {
- iwc.setMergePolicy(newMergePolicy());
- }
- IndexWriter w = new IndexWriter(dir, iwc);
- byte[] pointValue = new byte[3];
- byte[] uniquePointValue = new byte[3];
- random().nextBytes(uniquePointValue);
- final int numDocs =
- TEST_NIGHTLY ? atLeast(10000) : atLeast(500); // at night, make sure we have several leaves
- final boolean multiValues = random().nextBoolean();
- int totalValues = 0;
- for (int i = 0; i < numDocs; ++i) {
- Document doc = new Document();
- if (i == numDocs / 2) {
- totalValues++;
- doc.add(new BinaryPoint("f", uniquePointValue));
- } else {
- final int numValues = (multiValues) ? TestUtil.nextInt(random(), 2, 100) : 1;
- for (int j = 0; j < numValues; j++) {
- do {
- random().nextBytes(pointValue);
- } while (Arrays.equals(pointValue, uniquePointValue));
- doc.add(new BinaryPoint("f", pointValue));
- totalValues++;
- }
- }
- w.addDocument(doc);
- }
- w.forceMerge(1);
- final IndexReader r = DirectoryReader.open(w);
- w.close();
- final LeafReader lr = getOnlyLeafReader(r);
- PointValues points = lr.getPointValues("f");
-
- IntersectVisitor allPointsVisitor =
- new IntersectVisitor() {
- @Override
- public void visit(int docID, byte[] packedValue) throws IOException {}
-
- @Override
- public void visit(int docID) throws IOException {}
-
- @Override
- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- return Relation.CELL_INSIDE_QUERY;
- }
- };
-
- assertEquals(totalValues, points.estimatePointCount(allPointsVisitor));
- assertEquals(numDocs, points.estimateDocCount(allPointsVisitor));
-
- IntersectVisitor noPointsVisitor =
- new IntersectVisitor() {
- @Override
- public void visit(int docID, byte[] packedValue) throws IOException {}
-
- @Override
- public void visit(int docID) throws IOException {}
-
- @Override
- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- return Relation.CELL_OUTSIDE_QUERY;
- }
- };
-
- // Return 0 if no points match
- assertEquals(0, points.estimatePointCount(noPointsVisitor));
- assertEquals(0, points.estimateDocCount(noPointsVisitor));
-
- IntersectVisitor onePointMatchVisitor =
- new IntersectVisitor() {
- @Override
- public void visit(int docID, byte[] packedValue) throws IOException {}
-
- @Override
- public void visit(int docID) throws IOException {}
-
- @Override
- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- if (Arrays.compareUnsigned(uniquePointValue, 0, 3, maxPackedValue, 0, 3) > 0
- || Arrays.compareUnsigned(uniquePointValue, 0, 3, minPackedValue, 0, 3) < 0) {
- return Relation.CELL_OUTSIDE_QUERY;
- }
- return Relation.CELL_CROSSES_QUERY;
- }
- };
-
- // If only one point matches, then the point count is (maxPointsInLeafNode + 1) / 2
- // in general, or maybe 2x that if the point is a split value
- final long pointCount = points.estimatePointCount(onePointMatchVisitor);
- final long lastNodePointCount = totalValues % maxPointsInLeafNode;
- assertTrue(
- "" + pointCount,
- pointCount == (maxPointsInLeafNode + 1) / 2 // common case
- || pointCount == (lastNodePointCount + 1) / 2 // not fully populated leaf
- || pointCount == 2 * ((maxPointsInLeafNode + 1) / 2) // if the point is a split value
- || pointCount
- == ((maxPointsInLeafNode + 1) / 2)
- + ((lastNodePointCount + 1)
- / 2)); // if the point is a split value and one leaf is not fully populated
-
- final long docCount = points.estimateDocCount(onePointMatchVisitor);
-
- if (multiValues) {
- assertEquals(
- docCount,
- (long)
- (docCount
- * (1d
- - Math.pow(
- (numDocs - pointCount) / points.size(), points.size() / docCount))));
- } else {
- assertEquals(Math.min(pointCount, numDocs), docCount);
- }
- r.close();
- dir.close();
- }
-
- // The tree is always balanced in the N dims case, and leaves are
- // not all full so things are a bit different
- // TODO: clean up the math/estimation here rather than suppress so many warnings
- @SuppressWarnings({"NarrowCalculation", "LongDoubleConversion"})
- public void testEstimatePointCount2Dims() throws IOException {
- Directory dir = newDirectory();
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
- byte[][] pointValue = new byte[2][];
- pointValue[0] = new byte[3];
- pointValue[1] = new byte[3];
- byte[][] uniquePointValue = new byte[2][];
- uniquePointValue[0] = new byte[3];
- uniquePointValue[1] = new byte[3];
- random().nextBytes(uniquePointValue[0]);
- random().nextBytes(uniquePointValue[1]);
- final int numDocs =
- TEST_NIGHTLY
- ? atLeast(10000)
- : atLeast(1000); // in nightly, make sure we have several leaves
- final boolean multiValues = random().nextBoolean();
- int totalValues = 0;
- for (int i = 0; i < numDocs; ++i) {
- Document doc = new Document();
- if (i == numDocs / 2) {
- doc.add(new BinaryPoint("f", uniquePointValue));
- totalValues++;
- } else {
- final int numValues = (multiValues) ? TestUtil.nextInt(random(), 2, 100) : 1;
- for (int j = 0; j < numValues; j++) {
- do {
- random().nextBytes(pointValue[0]);
- random().nextBytes(pointValue[1]);
- } while (Arrays.equals(pointValue[0], uniquePointValue[0])
- || Arrays.equals(pointValue[1], uniquePointValue[1]));
- doc.add(new BinaryPoint("f", pointValue));
- totalValues++;
- }
- }
- w.addDocument(doc);
- }
- w.forceMerge(1);
- final IndexReader r = DirectoryReader.open(w);
- w.close();
- final LeafReader lr = getOnlyLeafReader(r);
- PointValues points = lr.getPointValues("f");
-
- IntersectVisitor allPointsVisitor =
- new IntersectVisitor() {
- @Override
- public void visit(int docID, byte[] packedValue) throws IOException {}
-
- @Override
- public void visit(int docID) throws IOException {}
-
- @Override
- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- return Relation.CELL_INSIDE_QUERY;
- }
- };
-
- assertEquals(totalValues, points.estimatePointCount(allPointsVisitor));
- assertEquals(numDocs, points.estimateDocCount(allPointsVisitor));
-
- IntersectVisitor noPointsVisitor =
- new IntersectVisitor() {
- @Override
- public void visit(int docID, byte[] packedValue) throws IOException {}
-
- @Override
- public void visit(int docID) throws IOException {}
-
- @Override
- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- return Relation.CELL_OUTSIDE_QUERY;
- }
- };
-
- // Return 0 if no points match
- assertEquals(0, points.estimatePointCount(noPointsVisitor));
- assertEquals(0, points.estimateDocCount(noPointsVisitor));
-
- IntersectVisitor onePointMatchVisitor =
- new IntersectVisitor() {
- @Override
- public void visit(int docID, byte[] packedValue) throws IOException {}
-
- @Override
- public void visit(int docID) throws IOException {}
-
- @Override
- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- for (int dim = 0; dim < 2; ++dim) {
- if (Arrays.compareUnsigned(
- uniquePointValue[dim], 0, 3, maxPackedValue, dim * 3, dim * 3 + 3)
- > 0
- || Arrays.compareUnsigned(
- uniquePointValue[dim], 0, 3, minPackedValue, dim * 3, dim * 3 + 3)
- < 0) {
- return Relation.CELL_OUTSIDE_QUERY;
- }
- }
- return Relation.CELL_CROSSES_QUERY;
- }
- };
-
- final long pointCount = points.estimatePointCount(onePointMatchVisitor);
- // With >1 dims, the tree is balanced
- long actualMaxPointsInLeafNode = points.size();
- while (actualMaxPointsInLeafNode > maxPointsInLeafNode) {
- actualMaxPointsInLeafNode = (actualMaxPointsInLeafNode + 1) / 2;
- }
- final long countPerFullLeaf = (actualMaxPointsInLeafNode + 1) / 2;
- final long countPerNotFullLeaf = (actualMaxPointsInLeafNode) / 2;
- assertTrue(
- pointCount + " vs " + actualMaxPointsInLeafNode,
- // common case, point in one leaf.
- pointCount >= countPerNotFullLeaf && pointCount <= countPerFullLeaf
- ||
- // one dimension is a split value
- pointCount >= 2 * countPerNotFullLeaf && pointCount <= 2 * countPerFullLeaf
- ||
- // both dimensions are split values
- pointCount >= 4 * countPerNotFullLeaf && pointCount <= 4 * countPerFullLeaf);
-
- final long docCount = points.estimateDocCount(onePointMatchVisitor);
- if (multiValues) {
- assertEquals(
- docCount,
- (long)
- (docCount
- * (1d
- - Math.pow(
- (numDocs - pointCount) / points.size(), points.size() / docCount))));
- } else {
- assertEquals(Math.min(pointCount, numDocs), docCount);
- }
- r.close();
- dir.close();
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/bkd/BKDWriter60.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/bkd/BKDWriter60.java
deleted file mode 100644
index c3381eff6b1..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/bkd/BKDWriter60.java
+++ /dev/null
@@ -1,2291 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene60.bkd;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-import java.util.function.IntFunction;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.MutablePointTree;
-import org.apache.lucene.index.MergeState;
-import org.apache.lucene.index.PointValues;
-import org.apache.lucene.index.PointValues.IntersectVisitor;
-import org.apache.lucene.index.PointValues.Relation;
-import org.apache.lucene.store.ByteBuffersDataOutput;
-import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.TrackingDirectoryWrapper;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.FixedBitSet;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.NumericUtils;
-import org.apache.lucene.util.PriorityQueue;
-import org.apache.lucene.util.bkd.BKDConfig;
-import org.apache.lucene.util.bkd.BKDRadixSelector;
-import org.apache.lucene.util.bkd.BKDReader;
-import org.apache.lucene.util.bkd.HeapPointWriter;
-import org.apache.lucene.util.bkd.MutablePointTreeReaderUtils;
-import org.apache.lucene.util.bkd.OfflinePointWriter;
-import org.apache.lucene.util.bkd.PointReader;
-import org.apache.lucene.util.bkd.PointValue;
-import org.apache.lucene.util.bkd.PointWriter;
-
-// TODO
-// - allow variable length byte[] (across docs and dims), but this is quite a bit more hairy
-// - we could also index "auto-prefix terms" here, and use better compression, and maybe only use
-// for the "fully contained" case so we'd
-// only index docIDs
-// - the index could be efficiently encoded as an FST, so we don't have wasteful
-// (monotonic) long[] leafBlockFPs; or we could use MonotonicLongValues ... but then
-// the index is already plenty small: 60M OSM points --> 1.1 MB with 128 points
-// per leaf, and you can reduce that by putting more points per leaf
-// - we could use threads while building; the higher nodes are very parallelizable
-
-/**
- * Recursively builds a block KD-tree to assign all incoming points in N-dim space to smaller and
- * smaller N-dim rectangles (cells) until the number of points in a given rectangle is &lt;= <code>
- * maxPointsInLeafNode</code>. The tree is fully balanced, which means the leaf nodes will have
- * between 50% and 100% of the requested <code>maxPointsInLeafNode</code>. Values that fall exactly
- * on a cell boundary may be in either cell.
- *
- * <p>The number of dimensions can be 1 to 8, but every byte[] value is fixed length.
- *
- * <p>This consumes heap during writing: it allocates a <code>Long[numLeaves]</code>, a <code>
- * byte[numLeaves*(1+bytesPerDim)]</code> and then uses up to the specified {@code maxMBSortInHeap}
- * heap space for writing.
- *
- * <p><b>NOTE</b>: This can write at most Integer.MAX_VALUE * <code>maxPointsInLeafNode</code> /
- * (1+bytesPerDim) total points.
- *
- * @lucene.experimental
- */
-public class BKDWriter60 implements Closeable {
-
- public static final String CODEC_NAME = "BKD";
- public static final int VERSION_START = 4; // version used by Lucene 7.0
- // public static final int VERSION_CURRENT = VERSION_START;
- public static final int VERSION_LEAF_STORES_BOUNDS = 5;
- public static final int VERSION_SELECTIVE_INDEXING = 6;
- public static final int VERSION_LOW_CARDINALITY_LEAVES = 7;
- public static final int VERSION_CURRENT = VERSION_LOW_CARDINALITY_LEAVES;
-
- /** How many bytes each docs takes in the fixed-width offline format */
- private final int bytesPerDoc;
-
- /** Default maximum number of point in each leaf block */
- public static final int DEFAULT_MAX_POINTS_IN_LEAF_NODE = 1024;
-
- /** Default maximum heap to use, before spilling to (slower) disk */
- public static final float DEFAULT_MAX_MB_SORT_IN_HEAP = 16.0f;
-
- /** Maximum number of index dimensions (2 * max index dimensions) */
- public static final int MAX_DIMS = 16;
-
- /** Maximum number of index dimensions */
- public static final int MAX_INDEX_DIMS = 8;
-
- /** Number of splits before we compute the exact bounding box of an inner node. */
- private static final int SPLITS_BEFORE_EXACT_BOUNDS = 4;
-
- /** How many dimensions we are storing at the leaf (data) nodes */
- protected final int numDataDims;
-
- /** How many dimensions we are indexing in the internal nodes */
- protected final int numIndexDims;
-
- /** How many bytes each value in each dimension takes. */
- protected final int bytesPerDim;
-
- /** numDataDims * bytesPerDim */
- protected final int packedBytesLength;
-
- /** numIndexDims * bytesPerDim */
- protected final int packedIndexBytesLength;
-
- final TrackingDirectoryWrapper tempDir;
- final String tempFileNamePrefix;
- final double maxMBSortInHeap;
-
- final byte[] scratchDiff;
- final byte[] scratch1;
- final byte[] scratch2;
- final BytesRef scratchBytesRef1 = new BytesRef();
- final BytesRef scratchBytesRef2 = new BytesRef();
- final int[] commonPrefixLengths;
-
- protected final FixedBitSet docsSeen;
-
- private PointWriter pointWriter;
- private boolean finished;
-
- private IndexOutput tempInput;
- protected final int maxPointsInLeafNode;
- private final int maxPointsSortInHeap;
-
- /** Minimum per-dim values, packed */
- protected final byte[] minPackedValue;
-
- /** Maximum per-dim values, packed */
- protected final byte[] maxPackedValue;
-
- protected long pointCount;
-
- /** An upper bound on how many points the caller will add (includes deletions) */
- private final long totalPointCount;
-
- private final int maxDoc;
-
- private final BKDConfig config;
-
- public BKDWriter60(
- int maxDoc,
- Directory tempDir,
- String tempFileNamePrefix,
- int numDataDims,
- int numIndexDims,
- int bytesPerDim,
- int maxPointsInLeafNode,
- double maxMBSortInHeap,
- long totalPointCount)
- throws IOException {
- verifyParams(numDataDims, numIndexDims, maxPointsInLeafNode, maxMBSortInHeap, totalPointCount);
- // We use tracking dir to deal with removing files on exception, so each place that
- // creates temp files doesn't need crazy try/finally/sucess logic:
- this.tempDir = new TrackingDirectoryWrapper(tempDir);
- this.tempFileNamePrefix = tempFileNamePrefix;
- this.maxPointsInLeafNode = maxPointsInLeafNode;
- this.numDataDims = numDataDims;
- this.numIndexDims = numIndexDims;
- this.bytesPerDim = bytesPerDim;
- this.config = new BKDConfig(numDataDims, numIndexDims, bytesPerDim, maxPointsInLeafNode);
- this.totalPointCount = totalPointCount;
- this.maxDoc = maxDoc;
- docsSeen = new FixedBitSet(maxDoc);
- packedBytesLength = numDataDims * bytesPerDim;
- packedIndexBytesLength = numIndexDims * bytesPerDim;
-
- scratchDiff = new byte[bytesPerDim];
- scratch1 = new byte[packedBytesLength];
- scratch2 = new byte[packedBytesLength];
- commonPrefixLengths = new int[numDataDims];
-
- minPackedValue = new byte[packedIndexBytesLength];
- maxPackedValue = new byte[packedIndexBytesLength];
-
- // dimensional values (numDims * bytesPerDim) + docID (int)
- bytesPerDoc = packedBytesLength + Integer.BYTES;
-
- // Maximum number of points we hold in memory at any time
- maxPointsSortInHeap = (int) ((maxMBSortInHeap * 1024 * 1024) / (bytesPerDoc));
-
- // Finally, we must be able to hold at least the leaf node in heap during build:
- if (maxPointsSortInHeap < maxPointsInLeafNode) {
- throw new IllegalArgumentException(
- "maxMBSortInHeap="
- + maxMBSortInHeap
- + " only allows for maxPointsSortInHeap="
- + maxPointsSortInHeap
- + ", but this is less than maxPointsInLeafNode="
- + maxPointsInLeafNode
- + "; either increase maxMBSortInHeap or decrease maxPointsInLeafNode");
- }
-
- this.maxMBSortInHeap = maxMBSortInHeap;
- }
-
- public static void verifyParams(
- int numDims,
- int numIndexDims,
- int maxPointsInLeafNode,
- double maxMBSortInHeap,
- long totalPointCount) {
- // We encode dim in a single byte in the splitPackedValues, but we only expose 4 bits for it
- // now, in case we want to use
- // remaining 4 bits for another purpose later
- if (numDims < 1 || numDims > MAX_DIMS) {
- throw new IllegalArgumentException(
- "numDims must be 1 .. " + MAX_DIMS + " (got: " + numDims + ")");
- }
- if (numIndexDims < 1 || numIndexDims > MAX_INDEX_DIMS) {
- throw new IllegalArgumentException(
- "numIndexDims must be 1 .. " + MAX_INDEX_DIMS + " (got: " + numIndexDims + ")");
- }
- if (numIndexDims > numDims) {
- throw new IllegalArgumentException(
- "numIndexDims cannot exceed numDims (" + numDims + ") (got: " + numIndexDims + ")");
- }
- if (maxPointsInLeafNode <= 0) {
- throw new IllegalArgumentException(
- "maxPointsInLeafNode must be > 0; got " + maxPointsInLeafNode);
- }
- if (maxPointsInLeafNode > ArrayUtil.MAX_ARRAY_LENGTH) {
- throw new IllegalArgumentException(
- "maxPointsInLeafNode must be <= ArrayUtil.MAX_ARRAY_LENGTH (= "
- + ArrayUtil.MAX_ARRAY_LENGTH
- + "); got "
- + maxPointsInLeafNode);
- }
- if (maxMBSortInHeap < 0.0) {
- throw new IllegalArgumentException(
- "maxMBSortInHeap must be >= 0.0 (got: " + maxMBSortInHeap + ")");
- }
- if (totalPointCount < 0) {
- throw new IllegalArgumentException(
- "totalPointCount must be >=0 (got: " + totalPointCount + ")");
- }
- }
-
- private void initPointWriter() throws IOException {
- assert pointWriter == null : "Point writer is already initialized";
- // total point count is an estimation but the final point count must be equal or lower to that
- // number.
- if (totalPointCount > maxPointsSortInHeap) {
- pointWriter = new OfflinePointWriter(config, tempDir, tempFileNamePrefix, "spill", 0);
- tempInput = ((OfflinePointWriter) pointWriter).out;
- } else {
- pointWriter = new HeapPointWriter(config, Math.toIntExact(totalPointCount));
- }
- }
-
- public void add(byte[] packedValue, int docID) throws IOException {
- if (packedValue.length != packedBytesLength) {
- throw new IllegalArgumentException(
- "packedValue should be length="
- + packedBytesLength
- + " (got: "
- + packedValue.length
- + ")");
- }
- if (pointCount >= totalPointCount) {
- throw new IllegalStateException(
- "totalPointCount="
- + totalPointCount
- + " was passed when we were created, but we just hit "
- + (pointCount + 1)
- + " values");
- }
- if (pointCount == 0) {
- initPointWriter();
- System.arraycopy(packedValue, 0, minPackedValue, 0, packedIndexBytesLength);
- System.arraycopy(packedValue, 0, maxPackedValue, 0, packedIndexBytesLength);
- } else {
- for (int dim = 0; dim < numIndexDims; dim++) {
- int offset = dim * bytesPerDim;
- if (Arrays.compareUnsigned(
- packedValue,
- offset,
- offset + bytesPerDim,
- minPackedValue,
- offset,
- offset + bytesPerDim)
- < 0) {
- System.arraycopy(packedValue, offset, minPackedValue, offset, bytesPerDim);
- }
- if (Arrays.compareUnsigned(
- packedValue,
- offset,
- offset + bytesPerDim,
- maxPackedValue,
- offset,
- offset + bytesPerDim)
- > 0) {
- System.arraycopy(packedValue, offset, maxPackedValue, offset, bytesPerDim);
- }
- }
- }
- pointWriter.append(packedValue, docID);
- pointCount++;
- docsSeen.set(docID);
- }
-
- /** How many points have been added so far */
- public long getPointCount() {
- return pointCount;
- }
-
- private static class MergeReader {
- private final PointValues.PointTree pointTree;
- private final int packedBytesLength;
- private final MergeState.DocMap docMap;
- private final MergeIntersectsVisitor mergeIntersectsVisitor;
-
- /** Which doc in this block we are up to */
- private int docBlockUpto;
-
- /** Current doc ID */
- public int docID;
-
- /** Current packed value */
- public final byte[] packedValue;
-
- public MergeReader(PointValues pointValues, MergeState.DocMap docMap) throws IOException {
- this.packedBytesLength = pointValues.getBytesPerDimension() * pointValues.getNumDimensions();
- this.pointTree = pointValues.getPointTree();
- this.mergeIntersectsVisitor = new MergeIntersectsVisitor(packedBytesLength);
- // move to first child of the tree and collect docs
- while (pointTree.moveToChild()) {}
- pointTree.visitDocValues(mergeIntersectsVisitor);
- this.docMap = docMap;
- this.packedValue = new byte[packedBytesLength];
- }
-
- public boolean next() throws IOException {
- // System.out.println("MR.next this=" + this);
- while (true) {
- if (docBlockUpto == mergeIntersectsVisitor.docsInBlock) {
- if (collectNextLeaf() == false) {
- assert mergeIntersectsVisitor.docsInBlock == 0;
- return false;
- }
- assert mergeIntersectsVisitor.docsInBlock > 0;
- docBlockUpto = 0;
- }
-
- final int index = docBlockUpto++;
- int oldDocID = mergeIntersectsVisitor.docIDs[index];
-
- int mappedDocID;
- if (docMap == null) {
- mappedDocID = oldDocID;
- } else {
- mappedDocID = docMap.get(oldDocID);
- }
-
- if (mappedDocID != -1) {
- // Not deleted!
- docID = mappedDocID;
- System.arraycopy(
- mergeIntersectsVisitor.packedValues,
- index * packedBytesLength,
- packedValue,
- 0,
- packedBytesLength);
- return true;
- }
- }
- }
-
- private boolean collectNextLeaf() throws IOException {
- assert pointTree.moveToChild() == false;
- mergeIntersectsVisitor.reset();
- do {
- if (pointTree.moveToSibling()) {
- // move to first child of this node and collect docs
- while (pointTree.moveToChild()) {}
- pointTree.visitDocValues(mergeIntersectsVisitor);
- return true;
- }
- } while (pointTree.moveToParent());
- return false;
- }
- }
-
- private static class MergeIntersectsVisitor implements IntersectVisitor {
-
- int docsInBlock = 0;
- byte[] packedValues;
- int[] docIDs;
- private final int packedBytesLength;
-
- MergeIntersectsVisitor(int packedBytesLength) {
- this.docIDs = new int[0];
- this.packedValues = new byte[0];
- this.packedBytesLength = packedBytesLength;
- }
-
- void reset() {
- docsInBlock = 0;
- }
-
- @Override
- public void grow(int count) {
- assert docsInBlock == 0;
- if (docIDs.length < count) {
- docIDs = ArrayUtil.grow(docIDs, count);
- int packedValuesSize = Math.toIntExact(docIDs.length * (long) packedBytesLength);
- if (packedValuesSize > ArrayUtil.MAX_ARRAY_LENGTH) {
- throw new IllegalStateException(
- "array length must be <= to "
- + ArrayUtil.MAX_ARRAY_LENGTH
- + " but was: "
- + packedValuesSize);
- }
- packedValues = ArrayUtil.growExact(packedValues, packedValuesSize);
- }
- }
-
- @Override
- public void visit(int docID) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void visit(int docID, byte[] packedValue) {
- System.arraycopy(
- packedValue, 0, packedValues, docsInBlock * packedBytesLength, packedBytesLength);
- docIDs[docsInBlock++] = docID;
- }
-
- @Override
- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- return Relation.CELL_CROSSES_QUERY;
- }
- }
-
- private static Comparator<MergeReader> mergeComparator(int bytesPerDim) {
- return ((Comparator<MergeReader>)
- (a, b) -> {
- assert a != b;
- return Arrays.compareUnsigned(
- a.packedValue, 0, bytesPerDim, b.packedValue, 0, bytesPerDim);
- })
- .thenComparingInt(mr -> mr.docID);
- }
-
- /**
- * Write a field from a {@link MutablePointTree}. This way of writing points is faster than
- * regular writes with {@link BKDWriter60#add} since there is opportunity for reordering points
- * before writing them to disk. This method does not use transient disk in order to reorder
- * points.
- */
- public long writeField(IndexOutput out, String fieldName, MutablePointTree reader)
- throws IOException {
- if (numDataDims == 1) {
- return writeField1Dim(out, fieldName, reader);
- } else {
- return writeFieldNDims(out, fieldName, reader);
- }
- }
-
- private void computePackedValueBounds(
- MutablePointTree values,
- int from,
- int to,
- byte[] minPackedValue,
- byte[] maxPackedValue,
- BytesRef scratch) {
- if (from == to) {
- return;
- }
- values.getValue(from, scratch);
- System.arraycopy(scratch.bytes, scratch.offset, minPackedValue, 0, packedIndexBytesLength);
- System.arraycopy(scratch.bytes, scratch.offset, maxPackedValue, 0, packedIndexBytesLength);
- for (int i = from + 1; i < to; ++i) {
- values.getValue(i, scratch);
- for (int dim = 0; dim < numIndexDims; dim++) {
- final int startOffset = dim * bytesPerDim;
- final int endOffset = startOffset + bytesPerDim;
- if (Arrays.compareUnsigned(
- scratch.bytes,
- scratch.offset + startOffset,
- scratch.offset + endOffset,
- minPackedValue,
- startOffset,
- endOffset)
- < 0) {
- System.arraycopy(
- scratch.bytes,
- scratch.offset + startOffset,
- minPackedValue,
- startOffset,
- bytesPerDim);
- } else if (Arrays.compareUnsigned(
- scratch.bytes,
- scratch.offset + startOffset,
- scratch.offset + endOffset,
- maxPackedValue,
- startOffset,
- endOffset)
- > 0) {
- System.arraycopy(
- scratch.bytes,
- scratch.offset + startOffset,
- maxPackedValue,
- startOffset,
- bytesPerDim);
- }
- }
- }
- }
-
- /* In the 2+D case, we recursively pick the split dimension, compute the
- * median value and partition other values around it. */
- private long writeFieldNDims(IndexOutput out, String fieldName, MutablePointTree values)
- throws IOException {
- if (pointCount != 0) {
- throw new IllegalStateException("cannot mix add and writeField");
- }
-
- // Catch user silliness:
- if (finished == true) {
- throw new IllegalStateException("already finished");
- }
-
- // Mark that we already finished:
- finished = true;
-
- long countPerLeaf = pointCount = values.size();
- long innerNodeCount = 1;
-
- while (countPerLeaf > maxPointsInLeafNode) {
- countPerLeaf = (countPerLeaf + 1) / 2;
- innerNodeCount *= 2;
- }
-
- int numLeaves = Math.toIntExact(innerNodeCount);
-
- checkMaxLeafNodeCount(numLeaves);
-
- final byte[] splitPackedValues = new byte[numLeaves * (bytesPerDim + 1)];
- final long[] leafBlockFPs = new long[numLeaves];
-
- // compute the min/max for this slice
- computePackedValueBounds(
- values, 0, Math.toIntExact(pointCount), minPackedValue, maxPackedValue, scratchBytesRef1);
- for (int i = 0; i < Math.toIntExact(pointCount); ++i) {
- docsSeen.set(values.getDocID(i));
- }
-
- final int[] parentSplits = new int[numIndexDims];
- build(
- 1,
- numLeaves,
- values,
- 0,
- Math.toIntExact(pointCount),
- out,
- minPackedValue.clone(),
- maxPackedValue.clone(),
- parentSplits,
- splitPackedValues,
- leafBlockFPs,
- new int[maxPointsInLeafNode]);
- assert Arrays.equals(parentSplits, new int[numIndexDims]);
-
- long indexFP = out.getFilePointer();
- writeIndex(out, Math.toIntExact(countPerLeaf), leafBlockFPs, splitPackedValues);
- return indexFP;
- }
-
- /* In the 1D case, we can simply sort points in ascending order and use the
- * same writing logic as we use at merge time. */
- private long writeField1Dim(IndexOutput out, String fieldName, MutablePointTree reader)
- throws IOException {
- MutablePointTreeReaderUtils.sort(config, maxDoc, reader, 0, Math.toIntExact(reader.size()));
-
- final OneDimensionBKDWriter oneDimWriter = new OneDimensionBKDWriter(out);
-
- reader.visitDocValues(
- new IntersectVisitor() {
-
- @Override
- public void visit(int docID, byte[] packedValue) throws IOException {
- oneDimWriter.add(packedValue, docID);
- }
-
- @Override
- public void visit(int docID) throws IOException {
- throw new IllegalStateException();
- }
-
- @Override
- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- return Relation.CELL_CROSSES_QUERY;
- }
- });
-
- return oneDimWriter.finish();
- }
-
- /**
- * More efficient bulk-add for incoming {@link BKDReader}s. This does a merge sort of the already
- * sorted values and currently only works when numDims==1. This returns -1 if all documents
- * containing dimensional values were deleted.
- */
- public long merge(IndexOutput out, List<MergeState.DocMap> docMaps, List<PointValues> readers)
- throws IOException {
- assert docMaps == null || readers.size() == docMaps.size();
-
- PriorityQueue<MergeReader> queue =
- PriorityQueue.usingComparator(readers.size(), mergeComparator(config.bytesPerDim()));
-
- for (int i = 0; i < readers.size(); i++) {
- PointValues pointValues = readers.get(i);
- assert pointValues.getNumDimensions() == config.numDims()
- && pointValues.getBytesPerDimension() == config.bytesPerDim()
- && pointValues.getNumIndexDimensions() == config.numIndexDims();
- MergeState.DocMap docMap;
- if (docMaps == null) {
- docMap = null;
- } else {
- docMap = docMaps.get(i);
- }
- MergeReader reader = new MergeReader(pointValues, docMap);
- if (reader.next()) {
- queue.add(reader);
- }
- }
-
- OneDimensionBKDWriter oneDimWriter = new OneDimensionBKDWriter(out);
-
- while (queue.size() != 0) {
- MergeReader reader = queue.top();
- // System.out.println("iter reader=" + reader);
-
- oneDimWriter.add(reader.packedValue, reader.docID);
-
- if (reader.next()) {
- queue.updateTop();
- } else {
- // This segment was exhausted
- queue.pop();
- }
- }
-
- return oneDimWriter.finish();
- }
-
- private class OneDimensionBKDWriter {
-
- final IndexOutput out;
- final List<Long> leafBlockFPs = new ArrayList<>();
- final List<byte[]> leafBlockStartValues = new ArrayList<>();
- final byte[] leafValues = new byte[maxPointsInLeafNode * packedBytesLength];
- final int[] leafDocs = new int[maxPointsInLeafNode];
- private long valueCount;
- private int leafCount;
- private int leafCardinality;
-
- OneDimensionBKDWriter(IndexOutput out) {
- if (numIndexDims != 1) {
- throw new UnsupportedOperationException("numIndexDims must be 1 but got " + numIndexDims);
- }
- if (pointCount != 0) {
- throw new IllegalStateException("cannot mix add and merge");
- }
-
- // Catch user silliness:
- if (finished == true) {
- throw new IllegalStateException("already finished");
- }
-
- // Mark that we already finished:
- finished = true;
-
- this.out = out;
-
- lastPackedValue = new byte[packedBytesLength];
- }
-
- // for asserts
- final byte[] lastPackedValue;
- private int lastDocID;
-
- void add(byte[] packedValue, int docID) throws IOException {
- assert valueInOrder(
- valueCount + leafCount, 0, lastPackedValue, packedValue, 0, docID, lastDocID);
-
- if (leafCount == 0
- || Arrays.mismatch(
- leafValues,
- (leafCount - 1) * bytesPerDim,
- leafCount * bytesPerDim,
- packedValue,
- 0,
- bytesPerDim)
- != -1) {
- leafCardinality++;
- }
- System.arraycopy(
- packedValue, 0, leafValues, leafCount * packedBytesLength, packedBytesLength);
- leafDocs[leafCount] = docID;
- docsSeen.set(docID);
- leafCount++;
-
- if (valueCount + leafCount > totalPointCount) {
- throw new IllegalStateException(
- "totalPointCount="
- + totalPointCount
- + " was passed when we were created, but we just hit "
- + (valueCount + leafCount)
- + " values");
- }
-
- if (leafCount == maxPointsInLeafNode) {
- // We write a block once we hit exactly the max count ... this is different from
- // when we write N > 1 dimensional points where we write between max/2 and max per leaf
- // block
- writeLeafBlock(leafCardinality);
- leafCardinality = 0;
- leafCount = 0;
- }
-
- assert (lastDocID = docID) >= 0; // only assign when asserts are enabled
- }
-
- public long finish() throws IOException {
- if (leafCount > 0) {
- writeLeafBlock(leafCardinality);
- leafCardinality = 0;
- leafCount = 0;
- }
-
- if (valueCount == 0) {
- return -1;
- }
-
- pointCount = valueCount;
-
- long indexFP = out.getFilePointer();
-
- int numInnerNodes = leafBlockStartValues.size();
-
- // System.out.println("BKDW: now rotate numInnerNodes=" + numInnerNodes + " leafBlockStarts="
- // + leafBlockStartValues.size());
-
- byte[] index = new byte[(1 + numInnerNodes) * (1 + bytesPerDim)];
- rotateToTree(1, 0, numInnerNodes, index, leafBlockStartValues);
- long[] arr = new long[leafBlockFPs.size()];
- for (int i = 0; i < leafBlockFPs.size(); i++) {
- arr[i] = leafBlockFPs.get(i);
- }
- writeIndex(out, maxPointsInLeafNode, arr, index);
- return indexFP;
- }
-
- private void writeLeafBlock(int leafCardinality) throws IOException {
- assert leafCount != 0;
- if (valueCount == 0) {
- System.arraycopy(leafValues, 0, minPackedValue, 0, packedIndexBytesLength);
- }
- System.arraycopy(
- leafValues,
- (leafCount - 1) * packedBytesLength,
- maxPackedValue,
- 0,
- packedIndexBytesLength);
-
- valueCount += leafCount;
-
- if (leafBlockFPs.size() > 0) {
- // Save the first (minimum) value in each leaf block except the first, to build the split
- // value index in the end:
- leafBlockStartValues.add(ArrayUtil.copyOfSubArray(leafValues, 0, packedBytesLength));
- }
- leafBlockFPs.add(out.getFilePointer());
- checkMaxLeafNodeCount(leafBlockFPs.size());
-
- // Find per-dim common prefix:
- int offset = (leafCount - 1) * packedBytesLength;
- int prefix =
- Arrays.mismatch(leafValues, 0, bytesPerDim, leafValues, offset, offset + bytesPerDim);
- if (prefix == -1) {
- prefix = bytesPerDim;
- }
-
- commonPrefixLengths[0] = prefix;
-
- writeLeafBlockDocs(out, leafDocs, 0, leafCount);
- writeCommonPrefixes(out, commonPrefixLengths, leafValues);
-
- scratchBytesRef1.length = packedBytesLength;
- scratchBytesRef1.bytes = leafValues;
-
- final IntFunction<BytesRef> packedValues =
- new IntFunction<>() {
- @Override
- public BytesRef apply(int i) {
- scratchBytesRef1.offset = packedBytesLength * i;
- return scratchBytesRef1;
- }
- };
- assert valuesInOrderAndBounds(
- leafCount,
- 0,
- ArrayUtil.copyOfSubArray(leafValues, 0, packedBytesLength),
- ArrayUtil.copyOfSubArray(
- leafValues, (leafCount - 1) * packedBytesLength, leafCount * packedBytesLength),
- packedValues,
- leafDocs,
- 0);
- writeLeafBlockPackedValues(
- out, commonPrefixLengths, leafCount, 0, packedValues, leafCardinality);
- }
- }
-
- // TODO: there must be a simpler way?
- private void rotateToTree(
- int nodeID, int offset, int count, byte[] index, List<byte[]> leafBlockStartValues) {
- // System.out.println("ROTATE: nodeID=" + nodeID + " offset=" + offset + " count=" + count + "
- // bpd=" + bytesPerDim + " index.length=" + index.length);
- if (count == 1) {
- // Leaf index node
- // System.out.println(" leaf index node");
- // System.out.println(" index[" + nodeID + "] = blockStartValues[" + offset + "]");
- System.arraycopy(
- leafBlockStartValues.get(offset), 0, index, nodeID * (1 + bytesPerDim) + 1, bytesPerDim);
- } else if (count > 1) {
- // Internal index node: binary partition of count
- int countAtLevel = 1;
- int totalCount = 0;
- while (true) {
- int countLeft = count - totalCount;
- // System.out.println(" cycle countLeft=" + countLeft + " coutAtLevel=" + countAtLevel);
- if (countLeft <= countAtLevel) {
- // This is the last level, possibly partially filled:
- int lastLeftCount = Math.min(countAtLevel / 2, countLeft);
- assert lastLeftCount >= 0;
- int leftHalf = (totalCount - 1) / 2 + lastLeftCount;
-
- int rootOffset = offset + leftHalf;
- /*
- System.out.println(" last left count " + lastLeftCount);
- System.out.println(" leftHalf " + leftHalf + " rightHalf=" + (count-leftHalf-1));
- System.out.println(" rootOffset=" + rootOffset);
- */
-
- System.arraycopy(
- leafBlockStartValues.get(rootOffset),
- 0,
- index,
- nodeID * (1 + bytesPerDim) + 1,
- bytesPerDim);
- // System.out.println(" index[" + nodeID + "] = blockStartValues[" + rootOffset + "]");
-
- // TODO: we could optimize/specialize, when we know it's simply fully balanced binary tree
- // under here, to save this while loop on each recursion
-
- // Recurse left
- rotateToTree(2 * nodeID, offset, leftHalf, index, leafBlockStartValues);
-
- // Recurse right
- rotateToTree(
- 2 * nodeID + 1, rootOffset + 1, count - leftHalf - 1, index, leafBlockStartValues);
- return;
- }
- totalCount += countAtLevel;
- countAtLevel *= 2;
- }
- } else {
- assert count == 0;
- }
- }
-
- // TODO: if we fixed each partition step to just record the file offset at the "split point", we
- // could probably handle variable length
- // encoding and not have our own ByteSequencesReader/Writer
-
- // useful for debugging:
- /*
- private void printPathSlice(String desc, PathSlice slice, int dim) throws IOException {
- System.out.println(" " + desc + " dim=" + dim + " count=" + slice.count + ":");
- try(PointReader r = slice.writer.getReader(slice.start, slice.count)) {
- int count = 0;
- while (r.next()) {
- byte[] v = r.packedValue();
- System.out.println(" " + count + ": " + new BytesRef(v, dim*bytesPerDim, bytesPerDim));
- count++;
- if (count == slice.count) {
- break;
- }
- }
- }
- }
- */
-
- private void checkMaxLeafNodeCount(int numLeaves) {
- if ((1 + bytesPerDim) * (long) numLeaves > ArrayUtil.MAX_ARRAY_LENGTH) {
- throw new IllegalStateException(
- "too many nodes; increase maxPointsInLeafNode (currently "
- + maxPointsInLeafNode
- + ") and reindex");
- }
- }
-
- /**
- * Writes the BKD tree to the provided {@link IndexOutput} and returns the file offset where index
- * was written.
- */
- public long finish(IndexOutput out) throws IOException {
- // System.out.println("\nBKDTreeWriter.finish pointCount=" + pointCount + " out=" + out + "
- // heapWriter=" + heapPointWriter);
-
- // TODO: specialize the 1D case? it's much faster at indexing time (no partitioning on
- // recurse...)
-
- // Catch user silliness:
- if (finished == true) {
- throw new IllegalStateException("already finished");
- }
-
- if (pointCount == 0) {
- throw new IllegalStateException("must index at least one point");
- }
-
- // mark as finished
- finished = true;
-
- pointWriter.close();
- BKDRadixSelector.PathSlice points = new BKDRadixSelector.PathSlice(pointWriter, 0, pointCount);
- // clean up pointers
- tempInput = null;
- pointWriter = null;
-
- long countPerLeaf = pointCount;
- long innerNodeCount = 1;
-
- while (countPerLeaf > maxPointsInLeafNode) {
- countPerLeaf = (countPerLeaf + 1) / 2;
- innerNodeCount *= 2;
- }
-
- int numLeaves = (int) innerNodeCount;
-
- checkMaxLeafNodeCount(numLeaves);
-
- // NOTE: we could save the 1+ here, to use a bit less heap at search time, but then we'd need a
- // somewhat costly check at each
- // step of the recursion to recompute the split dim:
-
- // Indexed by nodeID, but first (root) nodeID is 1. We do 1+ because the lead byte at each
- // recursion says which dim we split on.
- byte[] splitPackedValues = new byte[Math.toIntExact(numLeaves * (1 + bytesPerDim))];
-
- // +1 because leaf count is power of 2 (e.g. 8), and innerNodeCount is power of 2 minus 1 (e.g.
- // 7)
- long[] leafBlockFPs = new long[numLeaves];
-
- // Make sure the math above "worked":
- assert pointCount / numLeaves <= maxPointsInLeafNode
- : "pointCount="
- + pointCount
- + " numLeaves="
- + numLeaves
- + " maxPointsInLeafNode="
- + maxPointsInLeafNode;
-
- // We re-use the selector so we do not need to create an object every time.
- BKDRadixSelector radixSelector =
- new BKDRadixSelector(config, maxPointsSortInHeap, tempDir, tempFileNamePrefix);
-
- boolean success = false;
- try {
-
- final int[] parentSplits = new int[numIndexDims];
- build(
- 1,
- numLeaves,
- points,
- out,
- radixSelector,
- minPackedValue.clone(),
- maxPackedValue.clone(),
- parentSplits,
- splitPackedValues,
- leafBlockFPs,
- new int[maxPointsInLeafNode]);
- assert Arrays.equals(parentSplits, new int[numIndexDims]);
-
- // If no exception, we should have cleaned everything up:
- assert tempDir.getCreatedFiles().isEmpty();
- // System.out.println("write time: " + ((System.nanoTime() - t1) / (double)
- // TimeUnit.MILLISECONDS.toNanos(1)) + " ms");
-
- success = true;
- } finally {
- if (success == false) {
- IOUtils.deleteFilesIgnoringExceptions(tempDir, tempDir.getCreatedFiles());
- }
- }
-
- // System.out.println("Total nodes: " + innerNodeCount);
-
- // Write index:
- long indexFP = out.getFilePointer();
- writeIndex(out, Math.toIntExact(countPerLeaf), leafBlockFPs, splitPackedValues);
- return indexFP;
- }
-
- /** Packs the two arrays, representing a balanced binary tree, into a compact byte[] structure. */
- private byte[] packIndex(long[] leafBlockFPs, byte[] splitPackedValues) throws IOException {
-
- int numLeaves = leafBlockFPs.length;
-
- // Possibly rotate the leaf block FPs, if the index not fully balanced binary tree (only happens
- // if it was created by OneDimensionBKDWriter). In this case the leaf nodes may straddle the
- // two bottom
- // levels of the binary tree:
- if (numIndexDims == 1 && numLeaves > 1) {
- int levelCount = 2;
- while (true) {
- if (numLeaves >= levelCount && numLeaves <= 2 * levelCount) {
- int lastLevel = 2 * (numLeaves - levelCount);
- assert lastLevel >= 0;
- if (lastLevel != 0) {
- // Last level is partially filled, so we must rotate the leaf FPs to match. We do this
- // here, after loading
- // at read-time, so that we can still delta code them on disk at write:
- long[] newLeafBlockFPs = new long[numLeaves];
- System.arraycopy(
- leafBlockFPs, lastLevel, newLeafBlockFPs, 0, leafBlockFPs.length - lastLevel);
- System.arraycopy(
- leafBlockFPs, 0, newLeafBlockFPs, leafBlockFPs.length - lastLevel, lastLevel);
- leafBlockFPs = newLeafBlockFPs;
- }
- break;
- }
-
- levelCount *= 2;
- }
- }
-
- /* Reused while packing the index */
- ByteBuffersDataOutput writeBuffer = new ByteBuffersDataOutput();
-
- // This is the "file" we append the byte[] to:
- List<byte[]> blocks = new ArrayList<>();
- byte[] lastSplitValues = new byte[bytesPerDim * numIndexDims];
- // System.out.println("\npack index");
- int totalSize =
- recursePackIndex(
- writeBuffer,
- leafBlockFPs,
- splitPackedValues,
- 0l,
- blocks,
- 1,
- lastSplitValues,
- new boolean[numIndexDims],
- false);
-
- // Compact the byte[] blocks into single byte index:
- byte[] index = new byte[totalSize];
- int upto = 0;
- for (byte[] block : blocks) {
- System.arraycopy(block, 0, index, upto, block.length);
- upto += block.length;
- }
- assert upto == totalSize;
-
- return index;
- }
-
- /** Appends the current contents of writeBuffer as another block on the growing in-memory file */
- private int appendBlock(ByteBuffersDataOutput writeBuffer, List<byte[]> blocks)
- throws IOException {
- byte[] block = writeBuffer.toArrayCopy();
- blocks.add(block);
- writeBuffer.reset();
- return block.length;
- }
-
- /**
- * lastSplitValues is per-dimension split value previously seen; we use this to prefix-code the
- * split byte[] on each inner node
- */
- private int recursePackIndex(
- ByteBuffersDataOutput writeBuffer,
- long[] leafBlockFPs,
- byte[] splitPackedValues,
- long minBlockFP,
- List<byte[]> blocks,
- int nodeID,
- byte[] lastSplitValues,
- boolean[] negativeDeltas,
- boolean isLeft)
- throws IOException {
- if (nodeID >= leafBlockFPs.length) {
- int leafID = nodeID - leafBlockFPs.length;
- // System.out.println("recursePack leaf nodeID=" + nodeID);
-
- // In the unbalanced case it's possible the left most node only has one child:
- if (leafID < leafBlockFPs.length) {
- long delta = leafBlockFPs[leafID] - minBlockFP;
- if (isLeft) {
- assert delta == 0;
- return 0;
- } else {
- assert nodeID == 1 || delta > 0 : "nodeID=" + nodeID;
- writeBuffer.writeVLong(delta);
- return appendBlock(writeBuffer, blocks);
- }
- } else {
- return 0;
- }
- } else {
- long leftBlockFP;
- if (isLeft == false) {
- leftBlockFP = getLeftMostLeafBlockFP(leafBlockFPs, nodeID);
- long delta = leftBlockFP - minBlockFP;
- assert nodeID == 1 || delta > 0
- : "expected nodeID=1 or delta > 0; got nodeID=" + nodeID + " and delta=" + delta;
- writeBuffer.writeVLong(delta);
- } else {
- // The left tree's left most leaf block FP is always the minimal FP:
- leftBlockFP = minBlockFP;
- }
-
- int address = nodeID * (1 + bytesPerDim);
- int splitDim = splitPackedValues[address++] & 0xff;
-
- // System.out.println("recursePack inner nodeID=" + nodeID + " splitDim=" + splitDim + "
- // splitValue=" + new BytesRef(splitPackedValues, address, bytesPerDim));
-
- // find common prefix with last split value in this dim:
- int prefix =
- Arrays.mismatch(
- splitPackedValues,
- address,
- address + bytesPerDim,
- lastSplitValues,
- splitDim * bytesPerDim,
- splitDim * bytesPerDim + bytesPerDim);
- if (prefix == -1) {
- prefix = bytesPerDim;
- }
-
- // System.out.println("writeNodeData nodeID=" + nodeID + " splitDim=" + splitDim + " numDims="
- // + numDims + " bytesPerDim=" + bytesPerDim + " prefix=" + prefix);
-
- int firstDiffByteDelta;
- if (prefix < bytesPerDim) {
- // System.out.println(" delta byte cur=" +
- // Integer.toHexString(splitPackedValues[address+prefix]&0xFF) + " prev=" +
- // Integer.toHexString(lastSplitValues[splitDim * bytesPerDim + prefix]&0xFF) + " negated?="
- // + negativeDeltas[splitDim]);
- firstDiffByteDelta =
- (splitPackedValues[address + prefix] & 0xFF)
- - (lastSplitValues[splitDim * bytesPerDim + prefix] & 0xFF);
- if (negativeDeltas[splitDim]) {
- firstDiffByteDelta = -firstDiffByteDelta;
- }
- // System.out.println(" delta=" + firstDiffByteDelta);
- assert firstDiffByteDelta > 0;
- } else {
- firstDiffByteDelta = 0;
- }
-
- // pack the prefix, splitDim and delta first diff byte into a single vInt:
- int code = (firstDiffByteDelta * (1 + bytesPerDim) + prefix) * numIndexDims + splitDim;
-
- // System.out.println(" code=" + code);
- // System.out.println(" splitValue=" + new BytesRef(splitPackedValues, address,
- // bytesPerDim));
-
- writeBuffer.writeVInt(code);
-
- // write the split value, prefix coded vs. our parent's split value:
- int suffix = bytesPerDim - prefix;
- byte[] savSplitValue = new byte[suffix];
- if (suffix > 1) {
- writeBuffer.writeBytes(splitPackedValues, address + prefix + 1, suffix - 1);
- }
-
- byte[] cmp = lastSplitValues.clone();
-
- System.arraycopy(lastSplitValues, splitDim * bytesPerDim + prefix, savSplitValue, 0, suffix);
-
- // copy our split value into lastSplitValues for our children to prefix-code against
- System.arraycopy(
- splitPackedValues,
- address + prefix,
- lastSplitValues,
- splitDim * bytesPerDim + prefix,
- suffix);
-
- int numBytes = appendBlock(writeBuffer, blocks);
-
- // placeholder for left-tree numBytes; we need this so that at search time if we only need to
- // recurse into the right sub-tree we can
- // quickly seek to its starting point
- int idxSav = blocks.size();
- blocks.add(null);
-
- boolean savNegativeDelta = negativeDeltas[splitDim];
- negativeDeltas[splitDim] = true;
-
- int leftNumBytes =
- recursePackIndex(
- writeBuffer,
- leafBlockFPs,
- splitPackedValues,
- leftBlockFP,
- blocks,
- 2 * nodeID,
- lastSplitValues,
- negativeDeltas,
- true);
-
- if (nodeID * 2 < leafBlockFPs.length) {
- writeBuffer.writeVInt(leftNumBytes);
- } else {
- assert leftNumBytes == 0 : "leftNumBytes=" + leftNumBytes;
- }
-
- byte[] bytes2 = writeBuffer.toArrayCopy();
- writeBuffer.reset();
- int numBytes2 = bytes2.length;
- writeBuffer.reset();
- // replace our placeholder:
- blocks.set(idxSav, bytes2);
-
- negativeDeltas[splitDim] = false;
- int rightNumBytes =
- recursePackIndex(
- writeBuffer,
- leafBlockFPs,
- splitPackedValues,
- leftBlockFP,
- blocks,
- 2 * nodeID + 1,
- lastSplitValues,
- negativeDeltas,
- false);
-
- negativeDeltas[splitDim] = savNegativeDelta;
-
- // restore lastSplitValues to what caller originally passed us:
- System.arraycopy(savSplitValue, 0, lastSplitValues, splitDim * bytesPerDim + prefix, suffix);
-
- assert Arrays.equals(lastSplitValues, cmp);
-
- return numBytes + numBytes2 + leftNumBytes + rightNumBytes;
- }
- }
-
- private long getLeftMostLeafBlockFP(long[] leafBlockFPs, int nodeID) {
- // TODO: can we do this cheaper, e.g. a closed form solution instead of while loop? Or
- // change the recursion while packing the index to return this left-most leaf block FP
- // from each recursion instead?
- //
- // Still, the overall cost here is minor: this method's cost is O(log(N)), and while writing
- // we call it O(N) times (N = number of leaf blocks)
- while (nodeID < leafBlockFPs.length) {
- nodeID *= 2;
- }
- int leafID = nodeID - leafBlockFPs.length;
- long result = leafBlockFPs[leafID];
- if (result < 0) {
- throw new AssertionError(result + " for leaf " + leafID);
- }
- return result;
- }
-
- private void writeIndex(
- IndexOutput out, int countPerLeaf, long[] leafBlockFPs, byte[] splitPackedValues)
- throws IOException {
- byte[] packedIndex = packIndex(leafBlockFPs, splitPackedValues);
- writeIndex(out, countPerLeaf, leafBlockFPs.length, packedIndex);
- }
-
- private void writeIndex(IndexOutput out, int countPerLeaf, int numLeaves, byte[] packedIndex)
- throws IOException {
-
- CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT);
- out.writeVInt(numDataDims);
- out.writeVInt(numIndexDims);
- out.writeVInt(countPerLeaf);
- out.writeVInt(bytesPerDim);
-
- assert numLeaves > 0;
- out.writeVInt(numLeaves);
- out.writeBytes(minPackedValue, 0, packedIndexBytesLength);
- out.writeBytes(maxPackedValue, 0, packedIndexBytesLength);
-
- out.writeVLong(pointCount);
- out.writeVInt(docsSeen.cardinality());
- out.writeVInt(packedIndex.length);
- out.writeBytes(packedIndex, 0, packedIndex.length);
- }
-
- private void writeLeafBlockDocs(DataOutput out, int[] docIDs, int start, int count)
- throws IOException {
- assert count > 0 : "maxPointsInLeafNode=" + maxPointsInLeafNode;
- out.writeVInt(count);
- DocIdsWriter.writeDocIds(docIDs, start, count, out);
- }
-
- private void writeLeafBlockPackedValues(
- DataOutput out,
- int[] commonPrefixLengths,
- int count,
- int sortedDim,
- IntFunction<BytesRef> packedValues,
- int leafCardinality)
- throws IOException {
- int prefixLenSum = Arrays.stream(commonPrefixLengths).sum();
- if (prefixLenSum == packedBytesLength) {
- // all values in this block are equal
- out.writeByte((byte) -1);
- } else {
- assert commonPrefixLengths[sortedDim] < bytesPerDim;
- // estimate if storing the values with cardinality is cheaper than storing all values.
- int compressedByteOffset = sortedDim * bytesPerDim + commonPrefixLengths[sortedDim];
- int highCardinalityCost;
- int lowCardinalityCost;
- if (count == leafCardinality) {
- // all values in this block are different
- highCardinalityCost = 0;
- lowCardinalityCost = 1;
- } else {
- // compute cost of runLen compression
- int numRunLens = 0;
- for (int i = 0; i < count; ) {
- // do run-length compression on the byte at compressedByteOffset
- int runLen = runLen(packedValues, i, Math.min(i + 0xff, count), compressedByteOffset);
- assert runLen <= 0xff;
- numRunLens++;
- i += runLen;
- }
- // Add cost of runLen compression
- highCardinalityCost = count * (packedBytesLength - prefixLenSum - 1) + 2 * numRunLens;
- // +1 is the byte needed for storing the cardinality
- lowCardinalityCost = leafCardinality * (packedBytesLength - prefixLenSum + 1);
- }
- if (lowCardinalityCost <= highCardinalityCost) {
- out.writeByte((byte) -2);
- writeLowCardinalityLeafBlockPackedValues(out, commonPrefixLengths, count, packedValues);
- } else {
- out.writeByte((byte) sortedDim);
- writeHighCardinalityLeafBlockPackedValues(
- out, commonPrefixLengths, count, sortedDim, packedValues, compressedByteOffset);
- }
- }
- }
-
- private void writeLowCardinalityLeafBlockPackedValues(
- DataOutput out, int[] commonPrefixLengths, int count, IntFunction<BytesRef> packedValues)
- throws IOException {
- if (numIndexDims != 1) {
- writeActualBounds(out, commonPrefixLengths, count, packedValues);
- }
- BytesRef value = packedValues.apply(0);
- System.arraycopy(value.bytes, value.offset, scratch1, 0, packedBytesLength);
- int cardinality = 1;
- for (int i = 1; i < count; i++) {
- value = packedValues.apply(i);
- for (int dim = 0; dim < numDataDims; dim++) {
- final int start = dim * bytesPerDim + commonPrefixLengths[dim];
- final int end = dim * bytesPerDim + bytesPerDim;
- if (Arrays.mismatch(
- value.bytes, value.offset + start, value.offset + end, scratch1, start, end)
- != -1) {
- out.writeVInt(cardinality);
- for (int j = 0; j < numDataDims; j++) {
- out.writeBytes(
- scratch1,
- j * bytesPerDim + commonPrefixLengths[j],
- bytesPerDim - commonPrefixLengths[j]);
- }
- System.arraycopy(value.bytes, value.offset, scratch1, 0, packedBytesLength);
- cardinality = 1;
- break;
- } else if (dim == numDataDims - 1) {
- cardinality++;
- }
- }
- }
- out.writeVInt(cardinality);
- for (int i = 0; i < numDataDims; i++) {
- out.writeBytes(
- scratch1, i * bytesPerDim + commonPrefixLengths[i], bytesPerDim - commonPrefixLengths[i]);
- }
- }
-
- private void writeHighCardinalityLeafBlockPackedValues(
- DataOutput out,
- int[] commonPrefixLengths,
- int count,
- int sortedDim,
- IntFunction<BytesRef> packedValues,
- int compressedByteOffset)
- throws IOException {
- if (numIndexDims != 1) {
- writeActualBounds(out, commonPrefixLengths, count, packedValues);
- }
- commonPrefixLengths[sortedDim]++;
- for (int i = 0; i < count; ) {
- // do run-length compression on the byte at compressedByteOffset
- int runLen = runLen(packedValues, i, Math.min(i + 0xff, count), compressedByteOffset);
- assert runLen <= 0xff;
- BytesRef first = packedValues.apply(i);
- byte prefixByte = first.bytes[first.offset + compressedByteOffset];
- out.writeByte(prefixByte);
- out.writeByte((byte) runLen);
- writeLeafBlockPackedValuesRange(out, commonPrefixLengths, i, i + runLen, packedValues);
- i += runLen;
- assert i <= count;
- }
- }
-
- private void writeActualBounds(
- DataOutput out, int[] commonPrefixLengths, int count, IntFunction<BytesRef> packedValues)
- throws IOException {
- for (int dim = 0; dim < numIndexDims; ++dim) {
- int commonPrefixLength = commonPrefixLengths[dim];
- int suffixLength = bytesPerDim - commonPrefixLength;
- if (suffixLength > 0) {
- BytesRef[] minMax =
- computeMinMax(
- count, packedValues, dim * bytesPerDim + commonPrefixLength, suffixLength);
- BytesRef min = minMax[0];
- BytesRef max = minMax[1];
- out.writeBytes(min.bytes, min.offset, min.length);
- out.writeBytes(max.bytes, max.offset, max.length);
- }
- }
- }
-
- /**
- * Return an array that contains the min and max values for the [offset, offset+length] interval
- * of the given {@link BytesRef}s.
- */
- private static BytesRef[] computeMinMax(
- int count, IntFunction<BytesRef> packedValues, int offset, int length) {
- assert length > 0;
- BytesRefBuilder min = new BytesRefBuilder();
- BytesRefBuilder max = new BytesRefBuilder();
- BytesRef first = packedValues.apply(0);
- min.copyBytes(first.bytes, first.offset + offset, length);
- max.copyBytes(first.bytes, first.offset + offset, length);
- for (int i = 1; i < count; ++i) {
- BytesRef candidate = packedValues.apply(i);
- if (Arrays.compareUnsigned(
- min.bytes(),
- 0,
- length,
- candidate.bytes,
- candidate.offset + offset,
- candidate.offset + offset + length)
- > 0) {
- min.copyBytes(candidate.bytes, candidate.offset + offset, length);
- } else if (Arrays.compareUnsigned(
- max.bytes(),
- 0,
- length,
- candidate.bytes,
- candidate.offset + offset,
- candidate.offset + offset + length)
- < 0) {
- max.copyBytes(candidate.bytes, candidate.offset + offset, length);
- }
- }
- return new BytesRef[] {min.get(), max.get()};
- }
-
- private void writeLeafBlockPackedValuesRange(
- DataOutput out,
- int[] commonPrefixLengths,
- int start,
- int end,
- IntFunction<BytesRef> packedValues)
- throws IOException {
- for (int i = start; i < end; ++i) {
- BytesRef ref = packedValues.apply(i);
- assert ref.length == packedBytesLength;
-
- for (int dim = 0; dim < numDataDims; dim++) {
- int prefix = commonPrefixLengths[dim];
- out.writeBytes(ref.bytes, ref.offset + dim * bytesPerDim + prefix, bytesPerDim - prefix);
- }
- }
- }
-
- private static int runLen(
- IntFunction<BytesRef> packedValues, int start, int end, int byteOffset) {
- BytesRef first = packedValues.apply(start);
- byte b = first.bytes[first.offset + byteOffset];
- for (int i = start + 1; i < end; ++i) {
- BytesRef ref = packedValues.apply(i);
- byte b2 = ref.bytes[ref.offset + byteOffset];
- assert Byte.toUnsignedInt(b2) >= Byte.toUnsignedInt(b);
- if (b != b2) {
- return i - start;
- }
- }
- return end - start;
- }
-
- private void writeCommonPrefixes(DataOutput out, int[] commonPrefixes, byte[] packedValue)
- throws IOException {
- for (int dim = 0; dim < numDataDims; dim++) {
- out.writeVInt(commonPrefixes[dim]);
- // System.out.println(commonPrefixes[dim] + " of " + bytesPerDim);
- out.writeBytes(packedValue, dim * bytesPerDim, commonPrefixes[dim]);
- }
- }
-
- @Override
- public void close() throws IOException {
- finished = true;
- if (tempInput != null) {
- // NOTE: this should only happen on exception, e.g. caller calls close w/o calling finish:
- try {
- tempInput.close();
- } finally {
- tempDir.deleteFile(tempInput.getName());
- tempInput = null;
- }
- }
- }
-
- /**
- * Called on exception, to check whether the checksum is also corrupt in this source, and add that
- * information (checksum matched or didn't) as a suppressed exception.
- */
- private Error verifyChecksum(Throwable priorException, PointWriter writer) throws IOException {
- assert priorException != null;
-
- // TODO: we could improve this, to always validate checksum as we recurse, if we shared left and
- // right reader after recursing to children, and possibly within recursed children,
- // since all together they make a single pass through the file. But this is a sizable re-org,
- // and would mean leaving readers (IndexInputs) open for longer:
- if (writer instanceof OfflinePointWriter) {
- // We are reading from a temp file; go verify the checksum:
- String tempFileName = ((OfflinePointWriter) writer).name;
- if (tempDir.getCreatedFiles().contains(tempFileName)) {
- try (ChecksumIndexInput in = tempDir.openChecksumInput(tempFileName)) {
- CodecUtil.checkFooter(in, priorException);
- }
- }
- }
-
- // We are reading from heap; nothing to add:
- throw IOUtils.rethrowAlways(priorException);
- }
-
- /** Called only in assert */
- private boolean valueInBounds(
- BytesRef packedValue, byte[] minPackedValue, byte[] maxPackedValue) {
- for (int dim = 0; dim < numIndexDims; dim++) {
- int offset = bytesPerDim * dim;
- if (Arrays.compareUnsigned(
- packedValue.bytes,
- packedValue.offset + offset,
- packedValue.offset + offset + bytesPerDim,
- minPackedValue,
- offset,
- offset + bytesPerDim)
- < 0) {
- return false;
- }
- if (Arrays.compareUnsigned(
- packedValue.bytes,
- packedValue.offset + offset,
- packedValue.offset + offset + bytesPerDim,
- maxPackedValue,
- offset,
- offset + bytesPerDim)
- > 0) {
- return false;
- }
- }
-
- return true;
- }
-
- /**
- * Pick the next dimension to split.
- *
- * @param minPackedValue the min values for all dimensions
- * @param maxPackedValue the max values for all dimensions
- * @param parentSplits how many times each dim has been split on the parent levels
- * @return the dimension to split
- */
- protected int split(byte[] minPackedValue, byte[] maxPackedValue, int[] parentSplits) {
- // First look at whether there is a dimension that has split less than 2x less than
- // the dim that has most splits, and return it if there is such a dimension and it
- // does not only have equals values. This helps ensure all dimensions are indexed.
- int maxNumSplits = 0;
- for (int numSplits : parentSplits) {
- maxNumSplits = Math.max(maxNumSplits, numSplits);
- }
- for (int dim = 0; dim < numIndexDims; ++dim) {
- final int offset = dim * bytesPerDim;
- if (parentSplits[dim] < maxNumSplits / 2
- && Arrays.compareUnsigned(
- minPackedValue,
- offset,
- offset + bytesPerDim,
- maxPackedValue,
- offset,
- offset + bytesPerDim)
- != 0) {
- return dim;
- }
- }
-
- // Find which dim has the largest span so we can split on it:
- int splitDim = -1;
- for (int dim = 0; dim < numIndexDims; dim++) {
- NumericUtils.subtract(bytesPerDim, dim, maxPackedValue, minPackedValue, scratchDiff);
- if (splitDim == -1
- || Arrays.compareUnsigned(scratchDiff, 0, bytesPerDim, scratch1, 0, bytesPerDim) > 0) {
- System.arraycopy(scratchDiff, 0, scratch1, 0, bytesPerDim);
- splitDim = dim;
- }
- }
-
- // System.out.println("SPLIT: " + splitDim);
- return splitDim;
- }
-
- /** Pull a partition back into heap once the point count is low enough while recursing. */
- private HeapPointWriter switchToHeap(PointWriter source) throws IOException {
- int count = Math.toIntExact(source.count());
- try (PointReader reader = source.getReader(0, source.count());
- HeapPointWriter writer = new HeapPointWriter(config, count)) {
- for (int i = 0; i < count; i++) {
- boolean hasNext = reader.next();
- assert hasNext;
- writer.append(reader.pointValue());
- }
- source.destroy();
- return writer;
- } catch (Throwable t) {
- throw verifyChecksum(t, source);
- }
- }
-
- /* Recursively reorders the provided reader and writes the bkd-tree on the fly; this method is used
- * when we are writing a new segment directly from IndexWriter's indexing buffer (MutablePointsReader). */
- private void build(
- int nodeID,
- int leafNodeOffset,
- MutablePointTree reader,
- int from,
- int to,
- IndexOutput out,
- byte[] minPackedValue,
- byte[] maxPackedValue,
- int[] parentSplits,
- byte[] splitPackedValues,
- long[] leafBlockFPs,
- int[] spareDocIds)
- throws IOException {
-
- if (nodeID >= leafNodeOffset) {
- // leaf node
- final int count = to - from;
- assert count <= maxPointsInLeafNode;
-
- // Compute common prefixes
- Arrays.fill(commonPrefixLengths, bytesPerDim);
- reader.getValue(from, scratchBytesRef1);
- for (int i = from + 1; i < to; ++i) {
- reader.getValue(i, scratchBytesRef2);
- for (int dim = 0; dim < numDataDims; dim++) {
- final int offset = dim * bytesPerDim;
- int dimensionPrefixLength = commonPrefixLengths[dim];
- commonPrefixLengths[dim] =
- Arrays.mismatch(
- scratchBytesRef1.bytes,
- scratchBytesRef1.offset + offset,
- scratchBytesRef1.offset + offset + dimensionPrefixLength,
- scratchBytesRef2.bytes,
- scratchBytesRef2.offset + offset,
- scratchBytesRef2.offset + offset + dimensionPrefixLength);
- if (commonPrefixLengths[dim] == -1) {
- commonPrefixLengths[dim] = dimensionPrefixLength;
- }
- }
- }
-
- // Find the dimension that has the least number of unique bytes at commonPrefixLengths[dim]
- FixedBitSet[] usedBytes = new FixedBitSet[numDataDims];
- for (int dim = 0; dim < numDataDims; ++dim) {
- if (commonPrefixLengths[dim] < bytesPerDim) {
- usedBytes[dim] = new FixedBitSet(256);
- }
- }
- for (int i = from + 1; i < to; ++i) {
- for (int dim = 0; dim < numDataDims; dim++) {
- if (usedBytes[dim] != null) {
- byte b = reader.getByteAt(i, dim * bytesPerDim + commonPrefixLengths[dim]);
- usedBytes[dim].set(Byte.toUnsignedInt(b));
- }
- }
- }
- int sortedDim = 0;
- int sortedDimCardinality = Integer.MAX_VALUE;
- for (int dim = 0; dim < numDataDims; ++dim) {
- if (usedBytes[dim] != null) {
- final int cardinality = usedBytes[dim].cardinality();
- if (cardinality < sortedDimCardinality) {
- sortedDim = dim;
- sortedDimCardinality = cardinality;
- }
- }
- }
-
- // sort by sortedDim
- MutablePointTreeReaderUtils.sortByDim(
- config,
- sortedDim,
- commonPrefixLengths,
- reader,
- from,
- to,
- scratchBytesRef1,
- scratchBytesRef2);
-
- BytesRef comparator = scratchBytesRef1;
- BytesRef collector = scratchBytesRef2;
- reader.getValue(from, comparator);
- int leafCardinality = 1;
- for (int i = from + 1; i < to; ++i) {
- reader.getValue(i, collector);
- for (int dim = 0; dim < numDataDims; dim++) {
- final int start = dim * bytesPerDim + commonPrefixLengths[dim];
- final int end = dim * bytesPerDim + bytesPerDim;
- if (Arrays.mismatch(
- collector.bytes,
- collector.offset + start,
- collector.offset + end,
- comparator.bytes,
- comparator.offset + start,
- comparator.offset + end)
- != -1) {
- leafCardinality++;
- BytesRef scratch = collector;
- collector = comparator;
- comparator = scratch;
- break;
- }
- }
- }
- // Save the block file pointer:
- leafBlockFPs[nodeID - leafNodeOffset] = out.getFilePointer();
-
- // Write doc IDs
- int[] docIDs = spareDocIds;
- for (int i = from; i < to; ++i) {
- docIDs[i - from] = reader.getDocID(i);
- }
- // System.out.println("writeLeafBlock pos=" + out.getFilePointer());
- writeLeafBlockDocs(out, docIDs, 0, count);
-
- // Write the common prefixes:
- reader.getValue(from, scratchBytesRef1);
- System.arraycopy(
- scratchBytesRef1.bytes, scratchBytesRef1.offset, scratch1, 0, packedBytesLength);
- writeCommonPrefixes(out, commonPrefixLengths, scratch1);
-
- // Write the full values:
- IntFunction<BytesRef> packedValues =
- new IntFunction<>() {
- @Override
- public BytesRef apply(int i) {
- reader.getValue(from + i, scratchBytesRef1);
- return scratchBytesRef1;
- }
- };
- assert valuesInOrderAndBounds(
- count, sortedDim, minPackedValue, maxPackedValue, packedValues, docIDs, 0);
- writeLeafBlockPackedValues(
- out, commonPrefixLengths, count, sortedDim, packedValues, leafCardinality);
-
- } else {
- // inner node
-
- final int splitDim;
- // compute the split dimension and partition around it
- if (numIndexDims == 1) {
- splitDim = 0;
- } else {
- // for dimensions > 2 we recompute the bounds for the current inner node to help the
- // algorithm choose best
- // split dimensions. Because it is an expensive operation, the frequency we recompute the
- // bounds is given
- // by SPLITS_BEFORE_EXACT_BOUNDS.
- if (nodeID > 1
- && numIndexDims > 2
- && Arrays.stream(parentSplits).sum() % SPLITS_BEFORE_EXACT_BOUNDS == 0) {
- computePackedValueBounds(
- reader, from, to, minPackedValue, maxPackedValue, scratchBytesRef1);
- }
- splitDim = split(minPackedValue, maxPackedValue, parentSplits);
- }
-
- final int mid = (from + to + 1) >>> 1;
-
- int commonPrefixLen =
- Arrays.mismatch(
- minPackedValue,
- splitDim * bytesPerDim,
- splitDim * bytesPerDim + bytesPerDim,
- maxPackedValue,
- splitDim * bytesPerDim,
- splitDim * bytesPerDim + bytesPerDim);
- if (commonPrefixLen == -1) {
- commonPrefixLen = bytesPerDim;
- }
-
- MutablePointTreeReaderUtils.partition(
- config,
- maxDoc,
- splitDim,
- commonPrefixLen,
- reader,
- from,
- to,
- mid,
- scratchBytesRef1,
- scratchBytesRef2);
-
- // set the split value
- final int address = nodeID * (1 + bytesPerDim);
- splitPackedValues[address] = (byte) splitDim;
- reader.getValue(mid, scratchBytesRef1);
- System.arraycopy(
- scratchBytesRef1.bytes,
- scratchBytesRef1.offset + splitDim * bytesPerDim,
- splitPackedValues,
- address + 1,
- bytesPerDim);
-
- byte[] minSplitPackedValue =
- ArrayUtil.copyOfSubArray(minPackedValue, 0, packedIndexBytesLength);
- byte[] maxSplitPackedValue =
- ArrayUtil.copyOfSubArray(maxPackedValue, 0, packedIndexBytesLength);
- System.arraycopy(
- scratchBytesRef1.bytes,
- scratchBytesRef1.offset + splitDim * bytesPerDim,
- minSplitPackedValue,
- splitDim * bytesPerDim,
- bytesPerDim);
- System.arraycopy(
- scratchBytesRef1.bytes,
- scratchBytesRef1.offset + splitDim * bytesPerDim,
- maxSplitPackedValue,
- splitDim * bytesPerDim,
- bytesPerDim);
-
- // recurse
- parentSplits[splitDim]++;
- build(
- nodeID * 2,
- leafNodeOffset,
- reader,
- from,
- mid,
- out,
- minPackedValue,
- maxSplitPackedValue,
- parentSplits,
- splitPackedValues,
- leafBlockFPs,
- spareDocIds);
- build(
- nodeID * 2 + 1,
- leafNodeOffset,
- reader,
- mid,
- to,
- out,
- minSplitPackedValue,
- maxPackedValue,
- parentSplits,
- splitPackedValues,
- leafBlockFPs,
- spareDocIds);
- parentSplits[splitDim]--;
- }
- }
-
- private void computePackedValueBounds(
- BKDRadixSelector.PathSlice slice, byte[] minPackedValue, byte[] maxPackedValue)
- throws IOException {
- try (PointReader reader = slice.writer().getReader(slice.start(), slice.count())) {
- if (reader.next() == false) {
- return;
- }
- BytesRef value = reader.pointValue().packedValue();
- System.arraycopy(value.bytes, value.offset, minPackedValue, 0, packedIndexBytesLength);
- System.arraycopy(value.bytes, value.offset, maxPackedValue, 0, packedIndexBytesLength);
- while (reader.next()) {
- value = reader.pointValue().packedValue();
- for (int dim = 0; dim < numIndexDims; dim++) {
- final int startOffset = dim * bytesPerDim;
- final int endOffset = startOffset + bytesPerDim;
- if (Arrays.compareUnsigned(
- value.bytes,
- value.offset + startOffset,
- value.offset + endOffset,
- minPackedValue,
- startOffset,
- endOffset)
- < 0) {
- System.arraycopy(
- value.bytes, value.offset + startOffset, minPackedValue, startOffset, bytesPerDim);
- } else if (Arrays.compareUnsigned(
- value.bytes,
- value.offset + startOffset,
- value.offset + endOffset,
- maxPackedValue,
- startOffset,
- endOffset)
- > 0) {
- System.arraycopy(
- value.bytes, value.offset + startOffset, maxPackedValue, startOffset, bytesPerDim);
- }
- }
- }
- }
- }
-
- /**
- * The point writer contains the data that is going to be splitted using radix selection. /* This
- * method is used when we are merging previously written segments, in the numDims > 1 case.
- */
- private void build(
- int nodeID,
- int leafNodeOffset,
- BKDRadixSelector.PathSlice points,
- IndexOutput out,
- BKDRadixSelector radixSelector,
- byte[] minPackedValue,
- byte[] maxPackedValue,
- int[] parentSplits,
- byte[] splitPackedValues,
- long[] leafBlockFPs,
- int[] spareDocIds)
- throws IOException {
-
- if (nodeID >= leafNodeOffset) {
-
- // Leaf node: write block
- // We can write the block in any order so by default we write it sorted by the dimension that
- // has the
- // least number of unique bytes at commonPrefixLengths[dim], which makes compression more
- // efficient
- HeapPointWriter heapSource;
- if (points.writer() instanceof HeapPointWriter == false) {
- // Adversarial cases can cause this, e.g. merging big segments with most of the points
- // deleted
- heapSource = switchToHeap(points.writer());
- } else {
- heapSource = (HeapPointWriter) points.writer();
- }
-
- int from = Math.toIntExact(points.start());
- int to = Math.toIntExact(points.start() + points.count());
- // we store common prefix on scratch1
- computeCommonPrefixLength(heapSource, scratch1, from, to);
-
- int sortedDim = 0;
- int sortedDimCardinality = Integer.MAX_VALUE;
- FixedBitSet[] usedBytes = new FixedBitSet[numDataDims];
- for (int dim = 0; dim < numDataDims; ++dim) {
- if (commonPrefixLengths[dim] < bytesPerDim) {
- usedBytes[dim] = new FixedBitSet(256);
- }
- }
- // Find the dimension to compress
- for (int dim = 0; dim < numDataDims; dim++) {
- int prefix = commonPrefixLengths[dim];
- if (prefix < bytesPerDim) {
- int offset = dim * bytesPerDim;
- for (int i = from; i < to; ++i) {
- PointValue value = heapSource.getPackedValueSlice(i);
- BytesRef packedValue = value.packedValue();
- int bucket = packedValue.bytes[packedValue.offset + offset + prefix] & 0xff;
- usedBytes[dim].set(bucket);
- }
- int cardinality = usedBytes[dim].cardinality();
- if (cardinality < sortedDimCardinality) {
- sortedDim = dim;
- sortedDimCardinality = cardinality;
- }
- }
- }
-
- // sort the chosen dimension
- radixSelector.heapRadixSort(heapSource, from, to, sortedDim, commonPrefixLengths[sortedDim]);
- // compute cardinality
- int leafCardinality = heapSource.computeCardinality(from, to, commonPrefixLengths);
-
- // Save the block file pointer:
- leafBlockFPs[nodeID - leafNodeOffset] = out.getFilePointer();
- // System.out.println(" write leaf block @ fp=" + out.getFilePointer());
-
- // Write docIDs first, as their own chunk, so that at intersect time we can add all docIDs w/o
- // loading the values:
- int count = to - from;
- assert count > 0 : "nodeID=" + nodeID + " leafNodeOffset=" + leafNodeOffset;
- assert count <= spareDocIds.length : "count=" + count + " > length=" + spareDocIds.length;
- // Write doc IDs
- int[] docIDs = spareDocIds;
- for (int i = 0; i < count; i++) {
- docIDs[i] = heapSource.getPackedValueSlice(from + i).docID();
- }
- writeLeafBlockDocs(out, docIDs, 0, count);
-
- // TODO: minor opto: we don't really have to write the actual common prefixes, because
- // BKDReader on recursing can regenerate it for us
- // from the index, much like how terms dict does so from the FST:
-
- // Write the common prefixes:
- writeCommonPrefixes(out, commonPrefixLengths, scratch1);
-
- // Write the full values:
- IntFunction<BytesRef> packedValues =
- new IntFunction<>() {
- final BytesRef scratch = new BytesRef();
-
- {
- scratch.length = packedBytesLength;
- }
-
- @Override
- public BytesRef apply(int i) {
- PointValue value = heapSource.getPackedValueSlice(from + i);
- return value.packedValue();
- }
- };
- assert valuesInOrderAndBounds(
- count, sortedDim, minPackedValue, maxPackedValue, packedValues, docIDs, 0);
- writeLeafBlockPackedValues(
- out, commonPrefixLengths, count, sortedDim, packedValues, leafCardinality);
-
- } else {
- // Inner node: partition/recurse
-
- final int splitDim;
- if (numIndexDims == 1) {
- splitDim = 0;
- } else {
- // for dimensions > 2 we recompute the bounds for the current inner node to help the
- // algorithm choose best
- // split dimensions. Because it is an expensive operation, the frequency we recompute the
- // bounds is given
- // by SPLITS_BEFORE_EXACT_BOUNDS.
- if (nodeID > 1
- && numIndexDims > 2
- && Arrays.stream(parentSplits).sum() % SPLITS_BEFORE_EXACT_BOUNDS == 0) {
- computePackedValueBounds(points, minPackedValue, maxPackedValue);
- }
- splitDim = split(minPackedValue, maxPackedValue, parentSplits);
- }
-
- assert nodeID < splitPackedValues.length
- : "nodeID=" + nodeID + " splitValues.length=" + splitPackedValues.length;
-
- // How many points will be in the left tree:
- long rightCount = points.count() / 2;
- long leftCount = points.count() - rightCount;
-
- BKDRadixSelector.PathSlice[] slices = new BKDRadixSelector.PathSlice[2];
-
- int commonPrefixLen =
- Arrays.mismatch(
- minPackedValue,
- splitDim * bytesPerDim,
- splitDim * bytesPerDim + bytesPerDim,
- maxPackedValue,
- splitDim * bytesPerDim,
- splitDim * bytesPerDim + bytesPerDim);
- if (commonPrefixLen == -1) {
- commonPrefixLen = bytesPerDim;
- }
-
- byte[] splitValue =
- radixSelector.select(
- points,
- slices,
- points.start(),
- points.start() + points.count(),
- points.start() + leftCount,
- splitDim,
- commonPrefixLen);
-
- int address = nodeID * (1 + bytesPerDim);
- splitPackedValues[address] = (byte) splitDim;
- System.arraycopy(splitValue, 0, splitPackedValues, address + 1, bytesPerDim);
-
- byte[] minSplitPackedValue = new byte[packedIndexBytesLength];
- System.arraycopy(minPackedValue, 0, minSplitPackedValue, 0, packedIndexBytesLength);
-
- byte[] maxSplitPackedValue = new byte[packedIndexBytesLength];
- System.arraycopy(maxPackedValue, 0, maxSplitPackedValue, 0, packedIndexBytesLength);
-
- System.arraycopy(splitValue, 0, minSplitPackedValue, splitDim * bytesPerDim, bytesPerDim);
- System.arraycopy(splitValue, 0, maxSplitPackedValue, splitDim * bytesPerDim, bytesPerDim);
-
- parentSplits[splitDim]++;
- // Recurse on left tree:
- build(
- 2 * nodeID,
- leafNodeOffset,
- slices[0],
- out,
- radixSelector,
- minPackedValue,
- maxSplitPackedValue,
- parentSplits,
- splitPackedValues,
- leafBlockFPs,
- spareDocIds);
-
- // Recurse on right tree:
- build(
- 2 * nodeID + 1,
- leafNodeOffset,
- slices[1],
- out,
- radixSelector,
- minSplitPackedValue,
- maxPackedValue,
- parentSplits,
- splitPackedValues,
- leafBlockFPs,
- spareDocIds);
-
- parentSplits[splitDim]--;
- }
- }
-
- private void computeCommonPrefixLength(
- HeapPointWriter heapPointWriter, byte[] commonPrefix, int from, int to) {
- Arrays.fill(commonPrefixLengths, bytesPerDim);
- PointValue value = heapPointWriter.getPackedValueSlice(from);
- BytesRef packedValue = value.packedValue();
- for (int dim = 0; dim < numDataDims; dim++) {
- System.arraycopy(
- packedValue.bytes,
- packedValue.offset + dim * bytesPerDim,
- commonPrefix,
- dim * bytesPerDim,
- bytesPerDim);
- }
- for (int i = from + 1; i < to; i++) {
- value = heapPointWriter.getPackedValueSlice(i);
- packedValue = value.packedValue();
- for (int dim = 0; dim < numDataDims; dim++) {
- if (commonPrefixLengths[dim] != 0) {
- int j =
- Arrays.mismatch(
- commonPrefix,
- dim * bytesPerDim,
- dim * bytesPerDim + commonPrefixLengths[dim],
- packedValue.bytes,
- packedValue.offset + dim * bytesPerDim,
- packedValue.offset + dim * bytesPerDim + commonPrefixLengths[dim]);
- if (j != -1) {
- commonPrefixLengths[dim] = j;
- }
- }
- }
- }
- }
-
- // only called from assert
- private boolean valuesInOrderAndBounds(
- int count,
- int sortedDim,
- byte[] minPackedValue,
- byte[] maxPackedValue,
- IntFunction<BytesRef> values,
- int[] docs,
- int docsOffset)
- throws IOException {
- byte[] lastPackedValue = new byte[packedBytesLength];
- int lastDoc = -1;
- for (int i = 0; i < count; i++) {
- BytesRef packedValue = values.apply(i);
- assert packedValue.length == packedBytesLength;
- assert valueInOrder(
- i,
- sortedDim,
- lastPackedValue,
- packedValue.bytes,
- packedValue.offset,
- docs[docsOffset + i],
- lastDoc);
- lastDoc = docs[docsOffset + i];
-
- // Make sure this value does in fact fall within this leaf cell:
- assert valueInBounds(packedValue, minPackedValue, maxPackedValue);
- }
- return true;
- }
-
- // only called from assert
- private boolean valueInOrder(
- long ord,
- int sortedDim,
- byte[] lastPackedValue,
- byte[] packedValue,
- int packedValueOffset,
- int doc,
- int lastDoc) {
- int dimOffset = sortedDim * bytesPerDim;
- if (ord > 0) {
- int cmp =
- Arrays.compareUnsigned(
- lastPackedValue,
- dimOffset,
- dimOffset + bytesPerDim,
- packedValue,
- packedValueOffset + dimOffset,
- packedValueOffset + dimOffset + bytesPerDim);
- if (cmp > 0) {
- throw new AssertionError(
- "values out of order: last value="
- + new BytesRef(lastPackedValue)
- + " current value="
- + new BytesRef(packedValue, packedValueOffset, packedBytesLength)
- + " ord="
- + ord);
- }
- if (cmp == 0 && numDataDims > numIndexDims) {
- int dataOffset = numIndexDims * bytesPerDim;
- cmp =
- Arrays.compareUnsigned(
- lastPackedValue,
- dataOffset,
- packedBytesLength,
- packedValue,
- packedValueOffset + dataOffset,
- packedValueOffset + packedBytesLength);
- if (cmp > 0) {
- throw new AssertionError(
- "data values out of order: last value="
- + new BytesRef(lastPackedValue)
- + " current value="
- + new BytesRef(packedValue, packedValueOffset, packedBytesLength)
- + " ord="
- + ord);
- }
- }
- if (cmp == 0 && doc < lastDoc) {
- throw new AssertionError(
- "docs out of order: last doc=" + lastDoc + " current doc=" + doc + " ord=" + ord);
- }
- }
- System.arraycopy(packedValue, packedValueOffset, lastPackedValue, 0, packedBytesLength);
- return true;
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/bkd/DocIdsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/bkd/DocIdsWriter.java
deleted file mode 100644
index c460759cbe9..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene60/bkd/DocIdsWriter.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene60.bkd;
-
-import java.io.IOException;
-import org.apache.lucene.store.DataOutput;
-
-class DocIdsWriter {
-
- private DocIdsWriter() {}
-
- static void writeDocIds(int[] docIds, int start, int count, DataOutput out) throws IOException {
- // docs can be sorted either when all docs in a block have the same value
- // or when a segment is sorted
- boolean sorted = true;
- for (int i = 1; i < count; ++i) {
- if (docIds[start + i - 1] > docIds[start + i]) {
- sorted = false;
- break;
- }
- }
- if (sorted) {
- out.writeByte((byte) 0);
- int previous = 0;
- for (int i = 0; i < count; ++i) {
- int doc = docIds[start + i];
- out.writeVInt(doc - previous);
- previous = doc;
- }
- } else {
- long max = 0;
- for (int i = 0; i < count; ++i) {
- max |= Integer.toUnsignedLong(docIds[start + i]);
- }
- if (max <= 0xffffff) {
- out.writeByte((byte) 24);
- // write them the same way we are reading them.
- int i;
- for (i = 0; i < count - 7; i += 8) {
- int doc1 = docIds[start + i];
- int doc2 = docIds[start + i + 1];
- int doc3 = docIds[start + i + 2];
- int doc4 = docIds[start + i + 3];
- int doc5 = docIds[start + i + 4];
- int doc6 = docIds[start + i + 5];
- int doc7 = docIds[start + i + 6];
- int doc8 = docIds[start + i + 7];
- long l1 = (doc1 & 0xffffffL) << 40 | (doc2 & 0xffffffL) << 16 | ((doc3 >>> 8) & 0xffffL);
- long l2 =
- (doc3 & 0xffL) << 56
- | (doc4 & 0xffffffL) << 32
- | (doc5 & 0xffffffL) << 8
- | ((doc6 >> 16) & 0xffL);
- long l3 = (doc6 & 0xffffL) << 48 | (doc7 & 0xffffffL) << 24 | (doc8 & 0xffffffL);
- out.writeLong(l1);
- out.writeLong(l2);
- out.writeLong(l3);
- }
- for (; i < count; ++i) {
- out.writeShort((short) (docIds[start + i] >>> 8));
- out.writeByte((byte) docIds[start + i]);
- }
- } else {
- out.writeByte((byte) 32);
- for (int i = 0; i < count; ++i) {
- out.writeInt(docIds[start + i]);
- }
- }
- }
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene70/Lucene70RWSegmentInfoFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene70/Lucene70RWSegmentInfoFormat.java
deleted file mode 100644
index cf267f03c5a..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene70/Lucene70RWSegmentInfoFormat.java
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.lucene.backward_codecs.lucene70;
-
-import java.io.IOException;
-import java.util.Set;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.SortedNumericSelector;
-import org.apache.lucene.search.SortedNumericSortField;
-import org.apache.lucene.search.SortedSetSelector;
-import org.apache.lucene.search.SortedSetSortField;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.Version;
-
-/** Writable version of Lucene70SegmentInfoFormat for testing */
-public class Lucene70RWSegmentInfoFormat extends Lucene70SegmentInfoFormat {
-
- @Override
- public void write(Directory dir, SegmentInfo si, IOContext ioContext) throws IOException {
- final String fileName =
- IndexFileNames.segmentFileName(si.name, "", Lucene70SegmentInfoFormat.SI_EXTENSION);
-
- try (IndexOutput output = EndiannessReverserUtil.createOutput(dir, fileName, ioContext)) {
- // Only add the file once we've successfully created it, else IFD assert can trip:
- si.addFile(fileName);
- CodecUtil.writeIndexHeader(
- output,
- Lucene70SegmentInfoFormat.CODEC_NAME,
- Lucene70SegmentInfoFormat.VERSION_CURRENT,
- si.getId(),
- "");
-
- writeSegmentInfo(output, si);
-
- CodecUtil.writeFooter(output);
- }
- }
-
- private void writeSegmentInfo(IndexOutput output, SegmentInfo si) throws IOException {
- Version version = si.getVersion();
- if (version.major < 7) {
- throw new IllegalArgumentException(
- "invalid major version: should be >= 7 but got: " + version.major + " segment=" + si);
- }
- // Write the Lucene version that created this segment, since 3.1
- output.writeInt(version.major);
- output.writeInt(version.minor);
- output.writeInt(version.bugfix);
-
- // Write the min Lucene version that contributed docs to the segment, since 7.0
- if (si.getMinVersion() != null) {
- output.writeByte((byte) 1);
- Version minVersion = si.getMinVersion();
- output.writeInt(minVersion.major);
- output.writeInt(minVersion.minor);
- output.writeInt(minVersion.bugfix);
- } else {
- output.writeByte((byte) 0);
- }
-
- assert version.prerelease == 0;
- output.writeInt(si.maxDoc());
-
- output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO));
- output.writeMapOfStrings(si.getDiagnostics());
- Set<String> files = si.files();
- for (String file : files) {
- if (!IndexFileNames.parseSegmentName(file).equals(si.name)) {
- throw new IllegalArgumentException(
- "invalid files: expected segment=" + si.name + ", got=" + files);
- }
- }
- output.writeSetOfStrings(files);
- output.writeMapOfStrings(si.getAttributes());
-
- Sort indexSort = si.getIndexSort();
- int numSortFields = indexSort == null ? 0 : indexSort.getSort().length;
- output.writeVInt(numSortFields);
- for (int i = 0; i < numSortFields; ++i) {
- SortField sortField = indexSort.getSort()[i];
- SortField.Type sortType = sortField.getType();
- output.writeString(sortField.getField());
- int sortTypeID;
- switch (sortField.getType()) {
- case STRING:
- sortTypeID = 0;
- break;
- case LONG:
- sortTypeID = 1;
- break;
- case INT:
- sortTypeID = 2;
- break;
- case DOUBLE:
- sortTypeID = 3;
- break;
- case FLOAT:
- sortTypeID = 4;
- break;
- case CUSTOM:
- if (sortField instanceof SortedSetSortField) {
- sortTypeID = 5;
- sortType = SortField.Type.STRING;
- } else if (sortField instanceof SortedNumericSortField) {
- sortTypeID = 6;
- sortType = ((SortedNumericSortField) sortField).getNumericType();
- } else {
- throw new IllegalStateException("Unexpected SortedNumericSortField " + sortField);
- }
- break;
- case DOC:
- case REWRITEABLE:
- case STRING_VAL:
- case SCORE:
- default:
- throw new IllegalStateException("Unexpected sort type: " + sortField.getType());
- }
- output.writeVInt(sortTypeID);
- if (sortTypeID == 5) {
- SortedSetSortField ssf = (SortedSetSortField) sortField;
- if (ssf.getSelector() == SortedSetSelector.Type.MIN) {
- output.writeByte((byte) 0);
- } else if (ssf.getSelector() == SortedSetSelector.Type.MAX) {
- output.writeByte((byte) 1);
- } else if (ssf.getSelector() == SortedSetSelector.Type.MIDDLE_MIN) {
- output.writeByte((byte) 2);
- } else if (ssf.getSelector() == SortedSetSelector.Type.MIDDLE_MAX) {
- output.writeByte((byte) 3);
- } else {
- throw new IllegalStateException(
- "Unexpected SortedSetSelector type: " + ssf.getSelector());
- }
- } else if (sortTypeID == 6) {
- SortedNumericSortField snsf = (SortedNumericSortField) sortField;
- if (snsf.getNumericType() == SortField.Type.LONG) {
- output.writeByte((byte) 0);
- } else if (snsf.getNumericType() == SortField.Type.INT) {
- output.writeByte((byte) 1);
- } else if (snsf.getNumericType() == SortField.Type.DOUBLE) {
- output.writeByte((byte) 2);
- } else if (snsf.getNumericType() == SortField.Type.FLOAT) {
- output.writeByte((byte) 3);
- } else {
- throw new IllegalStateException(
- "Unexpected SortedNumericSelector type: " + snsf.getNumericType());
- }
- if (snsf.getSelector() == SortedNumericSelector.Type.MIN) {
- output.writeByte((byte) 0);
- } else if (snsf.getSelector() == SortedNumericSelector.Type.MAX) {
- output.writeByte((byte) 1);
- } else {
- throw new IllegalStateException(
- "Unexpected sorted numeric selector type: " + snsf.getSelector());
- }
- }
- output.writeByte((byte) (sortField.getReverse() ? 0 : 1));
-
- // write missing value
- Object missingValue = sortField.getMissingValue();
- if (missingValue == null) {
- output.writeByte((byte) 0);
- } else {
- switch (sortType) {
- case STRING:
- if (missingValue == SortField.STRING_LAST) {
- output.writeByte((byte) 1);
- } else if (missingValue == SortField.STRING_FIRST) {
- output.writeByte((byte) 2);
- } else {
- throw new AssertionError(
- "unrecognized missing value for STRING field \""
- + sortField.getField()
- + "\": "
- + missingValue);
- }
- break;
- case LONG:
- output.writeByte((byte) 1);
- output.writeLong(((Long) missingValue).longValue());
- break;
- case INT:
- output.writeByte((byte) 1);
- output.writeInt(((Integer) missingValue).intValue());
- break;
- case DOUBLE:
- output.writeByte((byte) 1);
- output.writeLong(Double.doubleToLongBits(((Double) missingValue).doubleValue()));
- break;
- case FLOAT:
- output.writeByte((byte) 1);
- output.writeInt(Float.floatToIntBits(((Float) missingValue).floatValue()));
- break;
- case CUSTOM:
- case DOC:
- case REWRITEABLE:
- case STRING_VAL:
- case SCORE:
- default:
- throw new IllegalStateException("Unexpected sort type: " + sortField.getType());
- }
- }
- }
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene70/TestLucene70SegmentInfoFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene70/TestLucene70SegmentInfoFormat.java
deleted file mode 100644
index 81482eff07c..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene70/TestLucene70SegmentInfoFormat.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.lucene.backward_codecs.lucene70;
-
-import org.apache.lucene.backward_codecs.lucene84.Lucene84RWCodec;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.tests.index.BaseSegmentInfoFormatTestCase;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-import org.apache.lucene.util.Version;
-
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestLucene70SegmentInfoFormat extends BaseSegmentInfoFormatTestCase {
-
- @Override
- protected Version[] getVersions() {
- return new Version[] {Version.fromBits(8, 4, 0)};
- }
-
- @Override
- protected Codec getCodec() {
- return new Lucene84RWCodec();
- }
-
- @Override
- protected boolean supportsHasBlocks() {
- return false;
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/BaseLucene80DocValuesFormatTestCase.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/BaseLucene80DocValuesFormatTestCase.java
deleted file mode 100644
index f77a950f0c1..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/BaseLucene80DocValuesFormatTestCase.java
+++ /dev/null
@@ -1,872 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene80;
-
-import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.function.LongSupplier;
-import java.util.function.Supplier;
-import org.apache.lucene.codecs.DocValuesFormat;
-import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
-import org.apache.lucene.document.BinaryDocValuesField;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.document.SortedDocValuesField;
-import org.apache.lucene.document.SortedNumericDocValuesField;
-import org.apache.lucene.document.SortedSetDocValuesField;
-import org.apache.lucene.document.StoredField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.SerialMergeScheduler;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.index.StoredFields;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.TermsEnum.SeekStatus;
-import org.apache.lucene.store.ByteBuffersDataInput;
-import org.apache.lucene.store.ByteBuffersDataOutput;
-import org.apache.lucene.store.ByteBuffersDirectory;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.tests.analysis.MockAnalyzer;
-import org.apache.lucene.tests.codecs.asserting.AssertingCodec;
-import org.apache.lucene.tests.index.LegacyBaseDocValuesFormatTestCase;
-import org.apache.lucene.tests.index.RandomIndexWriter;
-import org.apache.lucene.tests.util.TestUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.packed.PackedInts;
-
-/** Tests Lucene80DocValuesFormat */
-public abstract class BaseLucene80DocValuesFormatTestCase
- extends LegacyBaseDocValuesFormatTestCase {
-
- private static long dirSize(Directory d) throws IOException {
- long size = 0;
- for (String file : d.listAll()) {
- size += d.fileLength(file);
- }
- return size;
- }
-
- public void testUniqueValuesCompression() throws IOException {
- try (final Directory dir = new ByteBuffersDirectory()) {
- final IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- final IndexWriter iwriter = new IndexWriter(dir, iwc);
-
- final int uniqueValueCount = TestUtil.nextInt(random(), 1, 256);
- final List<Long> values = new ArrayList<>();
-
- final Document doc = new Document();
- final NumericDocValuesField dvf = new NumericDocValuesField("dv", 0);
- doc.add(dvf);
- for (int i = 0; i < 300; ++i) {
- final long value;
- if (values.size() < uniqueValueCount) {
- value = random().nextLong();
- values.add(value);
- } else {
- value = RandomPicks.randomFrom(random(), values);
- }
- dvf.setLongValue(value);
- iwriter.addDocument(doc);
- }
- iwriter.forceMerge(1);
- final long size1 = dirSize(dir);
- for (int i = 0; i < 20; ++i) {
- dvf.setLongValue(RandomPicks.randomFrom(random(), values));
- iwriter.addDocument(doc);
- }
- iwriter.forceMerge(1);
- final long size2 = dirSize(dir);
- // make sure the new longs did not cost 8 bytes each
- assertTrue(size2 < size1 + 8 * 20);
- }
- }
-
- public void testDateCompression() throws IOException {
- try (final Directory dir = new ByteBuffersDirectory()) {
- final IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- final IndexWriter iwriter = new IndexWriter(dir, iwc);
-
- final long base = 13; // prime
- final long day = 1000L * 60 * 60 * 24;
-
- final Document doc = new Document();
- final NumericDocValuesField dvf = new NumericDocValuesField("dv", 0);
- doc.add(dvf);
- for (int i = 0; i < 300; ++i) {
- dvf.setLongValue(base + random().nextInt(1000) * day);
- iwriter.addDocument(doc);
- }
- iwriter.forceMerge(1);
- final long size1 = dirSize(dir);
- for (int i = 0; i < 50; ++i) {
- dvf.setLongValue(base + random().nextInt(1000) * day);
- iwriter.addDocument(doc);
- }
- iwriter.forceMerge(1);
- final long size2 = dirSize(dir);
- // make sure the new longs costed less than if they had only been packed
- assertTrue(size2 < size1 + (PackedInts.bitsRequired(day) * 50) / 8);
- }
- }
-
- public void testSingleBigValueCompression() throws IOException {
- try (final Directory dir = new ByteBuffersDirectory()) {
- final IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- final IndexWriter iwriter = new IndexWriter(dir, iwc);
-
- final Document doc = new Document();
- final NumericDocValuesField dvf = new NumericDocValuesField("dv", 0);
- doc.add(dvf);
- for (int i = 0; i < 20000; ++i) {
- dvf.setLongValue(i & 1023);
- iwriter.addDocument(doc);
- }
- iwriter.forceMerge(1);
- final long size1 = dirSize(dir);
- dvf.setLongValue(Long.MAX_VALUE);
- iwriter.addDocument(doc);
- iwriter.forceMerge(1);
- final long size2 = dirSize(dir);
- // make sure the new value did not grow the bpv for every other value
- assertTrue(size2 < size1 + (20000 * (63 - 10)) / 8);
- }
- }
-
- // TODO: these big methods can easily blow up some of the other ram-hungry codecs...
- // for now just keep them here, as we want to test this for this format.
-
- public void testSortedSetVariableLengthBigVsStoredFields() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- int numDocs = TEST_NIGHTLY ? atLeast(100) : atLeast(10);
- doTestSortedSetVsStoredFields(numDocs, 1, 32766, 16, 100);
- }
- }
-
- @Nightly
- public void testSortedSetVariableLengthManyVsStoredFields() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- doTestSortedSetVsStoredFields(TestUtil.nextInt(random(), 1024, 2049), 1, 500, 16, 100);
- }
- }
-
- public void testSortedVariableLengthBigVsStoredFields() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- doTestSortedVsStoredFields(atLeast(100), 1d, 1, 32766);
- }
- }
-
- @Nightly
- public void testSortedVariableLengthManyVsStoredFields() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- doTestSortedVsStoredFields(TestUtil.nextInt(random(), 1024, 2049), 1d, 1, 500);
- }
- }
-
- @Nightly
- public void testTermsEnumFixedWidth() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- doTestTermsEnumRandom(
- TestUtil.nextInt(random(), 1025, 5121),
- () -> TestUtil.randomSimpleString(random(), 10, 10));
- }
- }
-
- @Nightly
- public void testTermsEnumVariableWidth() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- doTestTermsEnumRandom(
- TestUtil.nextInt(random(), 1025, 5121),
- () -> TestUtil.randomSimpleString(random(), 1, 500));
- }
- }
-
- @Nightly
- public void testTermsEnumRandomMany() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- doTestTermsEnumRandom(
- TestUtil.nextInt(random(), 1025, 8121),
- () -> TestUtil.randomSimpleString(random(), 1, 500));
- }
- }
-
- @Nightly
- public void testTermsEnumLongSharedPrefixes() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- doTestTermsEnumRandom(
- TestUtil.nextInt(random(), 1025, 5121),
- () -> {
- char[] chars = new char[random().nextInt(500)];
- Arrays.fill(chars, 'a');
- if (chars.length > 0) {
- chars[random().nextInt(chars.length)] = 'b';
- }
- return new String(chars);
- });
- }
- }
-
- public void testSparseDocValuesVsStoredFields() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- doTestSparseDocValuesVsStoredFields();
- }
- }
-
- private void doTestSparseDocValuesVsStoredFields() throws Exception {
- final long[] values = new long[TestUtil.nextInt(random(), 1, 500)];
- for (int i = 0; i < values.length; ++i) {
- values[i] = random().nextLong();
- }
-
- Directory dir = newFSDirectory(createTempDir());
- IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
- conf.setMergeScheduler(new SerialMergeScheduler());
- RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
-
- // sparse compression is only enabled if less than 1% of docs have a value
- final int avgGap = 100;
-
- final int numDocs = atLeast(200);
- for (int i = random().nextInt(avgGap * 2); i >= 0; --i) {
- writer.addDocument(new Document());
- }
- final int maxNumValuesPerDoc = random().nextBoolean() ? 1 : TestUtil.nextInt(random(), 2, 5);
- for (int i = 0; i < numDocs; ++i) {
- Document doc = new Document();
-
- // single-valued
- long docValue = values[random().nextInt(values.length)];
- doc.add(new NumericDocValuesField("numeric", docValue));
- doc.add(new SortedDocValuesField("sorted", new BytesRef(Long.toString(docValue))));
- doc.add(new BinaryDocValuesField("binary", new BytesRef(Long.toString(docValue))));
- doc.add(new StoredField("value", docValue));
-
- // multi-valued
- final int numValues = TestUtil.nextInt(random(), 1, maxNumValuesPerDoc);
- for (int j = 0; j < numValues; ++j) {
- docValue = values[random().nextInt(values.length)];
- doc.add(new SortedNumericDocValuesField("sorted_numeric", docValue));
- doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef(Long.toString(docValue))));
- doc.add(new StoredField("values", docValue));
- }
-
- writer.addDocument(doc);
-
- // add a gap
- for (int j = TestUtil.nextInt(random(), 0, avgGap * 2); j >= 0; --j) {
- writer.addDocument(new Document());
- }
- }
-
- if (random().nextBoolean()) {
- writer.forceMerge(1);
- }
-
- final IndexReader indexReader = writer.getReader();
- writer.close();
-
- for (LeafReaderContext context : indexReader.leaves()) {
- final LeafReader reader = context.reader();
- final NumericDocValues numeric = DocValues.getNumeric(reader, "numeric");
-
- final SortedDocValues sorted = DocValues.getSorted(reader, "sorted");
-
- final BinaryDocValues binary = DocValues.getBinary(reader, "binary");
-
- final SortedNumericDocValues sortedNumeric =
- DocValues.getSortedNumeric(reader, "sorted_numeric");
-
- final SortedSetDocValues sortedSet = DocValues.getSortedSet(reader, "sorted_set");
-
- StoredFields storedFields = reader.storedFields();
- for (int i = 0; i < reader.maxDoc(); ++i) {
- final Document doc = storedFields.document(i);
- final IndexableField valueField = doc.getField("value");
- final Long value = valueField == null ? null : valueField.numericValue().longValue();
-
- if (value == null) {
- assertTrue(numeric.docID() + " vs " + i, numeric.docID() < i);
- } else {
- assertEquals(i, numeric.nextDoc());
- assertEquals(i, binary.nextDoc());
- assertEquals(i, sorted.nextDoc());
- assertEquals(value.longValue(), numeric.longValue());
- assertTrue(sorted.ordValue() >= 0);
- assertEquals(new BytesRef(Long.toString(value)), sorted.lookupOrd(sorted.ordValue()));
- assertEquals(new BytesRef(Long.toString(value)), binary.binaryValue());
- }
-
- final IndexableField[] valuesFields = doc.getFields("values");
- if (valuesFields.length == 0) {
- assertTrue(sortedNumeric.docID() + " vs " + i, sortedNumeric.docID() < i);
- } else {
- final Set<Long> valueSet = new HashSet<>();
- for (IndexableField sf : valuesFields) {
- valueSet.add(sf.numericValue().longValue());
- }
-
- assertEquals(i, sortedNumeric.nextDoc());
- assertEquals(valuesFields.length, sortedNumeric.docValueCount());
- for (int j = 0; j < sortedNumeric.docValueCount(); ++j) {
- assertTrue(valueSet.contains(sortedNumeric.nextValue()));
- }
- assertEquals(i, sortedSet.nextDoc());
-
- assertEquals(valueSet.size(), sortedSet.docValueCount());
- for (int j = 0; j < sortedSet.docValueCount(); ++j) {
- long ord = sortedSet.nextOrd();
- assertTrue(valueSet.contains(Long.parseLong(sortedSet.lookupOrd(ord).utf8ToString())));
- }
- }
- }
- }
-
- indexReader.close();
- dir.close();
- }
-
- // TODO: try to refactor this and some termsenum tests into the base class.
- // to do this we need to fix the test class to get a DVF not a Codec so we can setup
- // the postings format correctly.
- private void doTestTermsEnumRandom(int numDocs, Supplier<String> valuesProducer)
- throws Exception {
- Directory dir = newFSDirectory(createTempDir());
- IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
- conf.setMergeScheduler(new SerialMergeScheduler());
- // set to duel against a codec which has ordinals:
- final PostingsFormat pf = TestUtil.getPostingsFormatWithOrds(random());
- final DocValuesFormat dv =
- ((PerFieldDocValuesFormat) getCodec().docValuesFormat())
- .getDocValuesFormatForField("random_field_name");
- conf.setCodec(
- new AssertingCodec() {
- @Override
- public PostingsFormat getPostingsFormatForField(String field) {
- return pf;
- }
-
- @Override
- public DocValuesFormat getDocValuesFormatForField(String field) {
- return dv;
- }
- });
- RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
-
- // index some docs
- for (int i = 0; i < numDocs; i++) {
- Document doc = new Document();
- Field idField = new StringField("id", Integer.toString(i), Field.Store.NO);
- doc.add(idField);
- int numValues = random().nextInt(17);
- // create a random list of strings
- List<String> values = new ArrayList<>();
- for (int v = 0; v < numValues; v++) {
- values.add(valuesProducer.get());
- }
-
- // add in any order to the indexed field
- ArrayList<String> unordered = new ArrayList<>(values);
- Collections.shuffle(unordered, random());
- for (String v : values) {
- doc.add(newStringField("indexed", v, Field.Store.NO));
- }
-
- // add in any order to the dv field
- ArrayList<String> unordered2 = new ArrayList<>(values);
- Collections.shuffle(unordered2, random());
- for (String v : unordered2) {
- doc.add(new SortedSetDocValuesField("dv", new BytesRef(v)));
- }
-
- writer.addDocument(doc);
- if (random().nextInt(31) == 0) {
- writer.commit();
- }
- }
-
- // delete some docs
- int numDeletions = random().nextInt(numDocs / 10);
- for (int i = 0; i < numDeletions; i++) {
- int id = random().nextInt(numDocs);
- writer.deleteDocuments(new Term("id", Integer.toString(id)));
- }
-
- // compare per-segment
- DirectoryReader ir = writer.getReader();
- for (LeafReaderContext context : ir.leaves()) {
- LeafReader r = context.reader();
- Terms terms = r.terms("indexed");
- if (terms != null) {
- SortedSetDocValues ssdv = r.getSortedSetDocValues("dv");
- assertEquals(terms.size(), ssdv.getValueCount());
- TermsEnum expected = terms.iterator();
- TermsEnum actual = r.getSortedSetDocValues("dv").termsEnum();
- assertEquals(terms.size(), expected, actual);
-
- doTestSortedSetEnumAdvanceIndependently(ssdv);
- }
- }
- ir.close();
-
- writer.forceMerge(1);
-
- // now compare again after the merge
- ir = writer.getReader();
- LeafReader ar = getOnlyLeafReader(ir);
- Terms terms = ar.terms("indexed");
- if (terms != null) {
- assertEquals(terms.size(), ar.getSortedSetDocValues("dv").getValueCount());
- TermsEnum expected = terms.iterator();
- TermsEnum actual = ar.getSortedSetDocValues("dv").termsEnum();
- assertEquals(terms.size(), expected, actual);
- }
- ir.close();
-
- writer.close();
- dir.close();
- }
-
- private void assertEquals(long numOrds, TermsEnum expected, TermsEnum actual) throws Exception {
- BytesRef ref;
-
- // sequential next() through all terms
- while ((ref = expected.next()) != null) {
- assertEquals(ref, actual.next());
- assertEquals(expected.ord(), actual.ord());
- assertEquals(expected.term(), actual.term());
- }
- assertNull(actual.next());
-
- // sequential seekExact(ord) through all terms
- for (long i = 0; i < numOrds; i++) {
- expected.seekExact(i);
- actual.seekExact(i);
- assertEquals(expected.ord(), actual.ord());
- assertEquals(expected.term(), actual.term());
- }
-
- // sequential seekExact(BytesRef) through all terms
- for (long i = 0; i < numOrds; i++) {
- expected.seekExact(i);
- assertTrue(actual.seekExact(expected.term()));
- assertEquals(expected.ord(), actual.ord());
- assertEquals(expected.term(), actual.term());
- }
-
- // sequential seekCeil(BytesRef) through all terms
- for (long i = 0; i < numOrds; i++) {
- expected.seekExact(i);
- assertEquals(SeekStatus.FOUND, actual.seekCeil(expected.term()));
- assertEquals(expected.ord(), actual.ord());
- assertEquals(expected.term(), actual.term());
- }
-
- // random seekExact(ord)
- for (long i = 0; i < numOrds; i++) {
- long randomOrd = TestUtil.nextLong(random(), 0, numOrds - 1);
- expected.seekExact(randomOrd);
- actual.seekExact(randomOrd);
- assertEquals(expected.ord(), actual.ord());
- assertEquals(expected.term(), actual.term());
- }
-
- // random seekExact(BytesRef)
- for (long i = 0; i < numOrds; i++) {
- long randomOrd = TestUtil.nextLong(random(), 0, numOrds - 1);
- expected.seekExact(randomOrd);
- actual.seekExact(expected.term());
- assertEquals(expected.ord(), actual.ord());
- assertEquals(expected.term(), actual.term());
- }
-
- // random seekCeil(BytesRef)
- for (long i = 0; i < numOrds; i++) {
- BytesRef target = new BytesRef(TestUtil.randomUnicodeString(random()));
- SeekStatus expectedStatus = expected.seekCeil(target);
- assertEquals(expectedStatus, actual.seekCeil(target));
- if (expectedStatus != SeekStatus.END) {
- assertEquals(expected.ord(), actual.ord());
- assertEquals(expected.term(), actual.term());
- }
- }
- }
-
- @Nightly
- public void testSortedSetAroundBlockSize() throws IOException {
- final int frontier = 1 << Lucene80DocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
- for (int maxDoc = frontier - 1; maxDoc <= frontier + 1; ++maxDoc) {
- final Directory dir = newDirectory();
- IndexWriter w =
- new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(newLogMergePolicy()));
- ByteBuffersDataOutput out = new ByteBuffersDataOutput();
- Document doc = new Document();
- SortedSetDocValuesField field1 = new SortedSetDocValuesField("sset", new BytesRef());
- doc.add(field1);
- SortedSetDocValuesField field2 = new SortedSetDocValuesField("sset", new BytesRef());
- doc.add(field2);
- for (int i = 0; i < maxDoc; ++i) {
- BytesRef s1 = new BytesRef(TestUtil.randomSimpleString(random(), 2));
- BytesRef s2 = new BytesRef(TestUtil.randomSimpleString(random(), 2));
- field1.setBytesValue(s1);
- field2.setBytesValue(s2);
- w.addDocument(doc);
- Set<BytesRef> set = new TreeSet<>(Arrays.asList(s1, s2));
- out.writeVInt(set.size());
- for (BytesRef ref : set) {
- out.writeVInt(ref.length);
- out.writeBytes(ref.bytes, ref.offset, ref.length);
- }
- }
-
- w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- w.close();
- LeafReader sr = getOnlyLeafReader(r);
- assertEquals(maxDoc, sr.maxDoc());
- SortedSetDocValues values = sr.getSortedSetDocValues("sset");
- assertNotNull(values);
- ByteBuffersDataInput in = out.toDataInput();
- BytesRefBuilder b = new BytesRefBuilder();
- for (int i = 0; i < maxDoc; ++i) {
- assertEquals(i, values.nextDoc());
- final int numValues = in.readVInt();
- assertEquals(numValues, values.docValueCount());
-
- for (int j = 0; j < numValues; ++j) {
- b.setLength(in.readVInt());
- b.grow(b.length());
- in.readBytes(b.bytes(), 0, b.length());
- assertEquals(b.get(), values.lookupOrd(values.nextOrd()));
- }
- }
- r.close();
- dir.close();
- }
- }
-
- @Nightly
- public void testSortedNumericAroundBlockSize() throws IOException {
- final int frontier = 1 << Lucene80DocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
- for (int maxDoc = frontier - 1; maxDoc <= frontier + 1; ++maxDoc) {
- final Directory dir = newDirectory();
- IndexWriter w =
- new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(newLogMergePolicy()));
- ByteBuffersDataOutput buffer = new ByteBuffersDataOutput();
-
- Document doc = new Document();
- SortedNumericDocValuesField field1 = new SortedNumericDocValuesField("snum", 0L);
- doc.add(field1);
- SortedNumericDocValuesField field2 = new SortedNumericDocValuesField("snum", 0L);
- doc.add(field2);
- for (int i = 0; i < maxDoc; ++i) {
- long s1 = random().nextInt(100);
- long s2 = random().nextInt(100);
- field1.setLongValue(s1);
- field2.setLongValue(s2);
- w.addDocument(doc);
- buffer.writeVLong(Math.min(s1, s2));
- buffer.writeVLong(Math.max(s1, s2));
- }
-
- w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- w.close();
- LeafReader sr = getOnlyLeafReader(r);
- assertEquals(maxDoc, sr.maxDoc());
- SortedNumericDocValues values = sr.getSortedNumericDocValues("snum");
- assertNotNull(values);
- ByteBuffersDataInput dataInput = buffer.toDataInput();
- for (int i = 0; i < maxDoc; ++i) {
- assertEquals(i, values.nextDoc());
- assertEquals(2, values.docValueCount());
- assertEquals(dataInput.readVLong(), values.nextValue());
- assertEquals(dataInput.readVLong(), values.nextValue());
- }
- r.close();
- dir.close();
- }
- }
-
- @Nightly
- public void testSortedNumericBlocksOfVariousBitsPerValue() throws Exception {
- doTestSortedNumericBlocksOfVariousBitsPerValue(() -> TestUtil.nextInt(random(), 1, 3));
- }
-
- @Nightly
- public void testSparseSortedNumericBlocksOfVariousBitsPerValue() throws Exception {
- doTestSortedNumericBlocksOfVariousBitsPerValue(() -> TestUtil.nextInt(random(), 0, 2));
- }
-
- @Nightly
- public void testNumericBlocksOfVariousBitsPerValue() throws Exception {
- doTestSparseNumericBlocksOfVariousBitsPerValue(1);
- }
-
- @Nightly
- public void testSparseNumericBlocksOfVariousBitsPerValue() throws Exception {
- doTestSparseNumericBlocksOfVariousBitsPerValue(random().nextDouble());
- }
-
- // The LUCENE-8585 jump-tables enables O(1) skipping of IndexedDISI blocks, DENSE block lookup
- // and numeric multi blocks. This test focuses on testing these jumps.
- @Nightly
- public void testNumericFieldJumpTables() throws Exception {
- // IndexedDISI block skipping only activated if target >= current+2, so we need at least 5
- // blocks to
- // trigger consecutive block skips
- final int maxDoc = atLeast(5 * 65536);
-
- Directory dir = newDirectory();
- IndexWriter iw = createFastIndexWriter(dir, maxDoc);
-
- Field idField = newStringField("id", "", Field.Store.NO);
- Field storedField = newStringField("stored", "", Field.Store.YES);
- Field dvField = new NumericDocValuesField("dv", 0);
-
- for (int i = 0; i < maxDoc; i++) {
- Document doc = new Document();
- idField.setStringValue(Integer.toBinaryString(i));
- doc.add(idField);
- if (random().nextInt(100) > 10) { // Skip 10% to make DENSE blocks
- int value = random().nextInt(100000);
- storedField.setStringValue(Integer.toString(value));
- doc.add(storedField);
- dvField.setLongValue(value);
- doc.add(dvField);
- }
- iw.addDocument(doc);
- }
- iw.flush();
- iw.forceMerge(1, true); // Single segment to force large enough structures
- iw.commit();
- iw.close();
-
- assertDVIterate(dir);
- assertDVAdvance(
- dir, rarely() ? 1 : 7); // 1 is heavy (~20 s), so we do it rarely. 7 is a lot faster (8 s)
-
- dir.close();
- }
-
- private IndexWriter createFastIndexWriter(Directory dir, int maxBufferedDocs) throws IOException {
- IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
- conf.setMaxBufferedDocs(maxBufferedDocs);
- conf.setRAMBufferSizeMB(-1);
- conf.setMergePolicy(newLogMergePolicy());
- conf.getCodec().compoundFormat().setShouldUseCompoundFile(random().nextBoolean());
- return new IndexWriter(dir, conf);
- }
-
- private static LongSupplier blocksOfVariousBPV() {
- final long mul = TestUtil.nextInt(random(), 1, 100);
- final long min = random().nextInt();
- return new LongSupplier() {
- int i = Lucene80DocValuesFormat.NUMERIC_BLOCK_SIZE;
- int maxDelta;
-
- @Override
- public long getAsLong() {
- if (i == Lucene80DocValuesFormat.NUMERIC_BLOCK_SIZE) {
- maxDelta = 1 << random().nextInt(5);
- i = 0;
- }
- i++;
- return min + mul * random().nextInt(maxDelta);
- }
- };
- }
-
- private void doTestSortedNumericBlocksOfVariousBitsPerValue(LongSupplier counts)
- throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
- conf.setMaxBufferedDocs(atLeast(Lucene80DocValuesFormat.NUMERIC_BLOCK_SIZE));
- conf.setRAMBufferSizeMB(-1);
- // so Lucene docids are predictable / stay in order
- conf.setMergePolicy(newLogMergePolicy());
- conf.getCodec().compoundFormat().setShouldUseCompoundFile(random().nextBoolean());
- IndexWriter writer = new IndexWriter(dir, conf);
-
- final int numDocs = atLeast(Lucene80DocValuesFormat.NUMERIC_BLOCK_SIZE * 3);
- final LongSupplier values = blocksOfVariousBPV();
- List<long[]> writeDocValues = new ArrayList<>();
- for (int i = 0; i < numDocs; i++) {
- Document doc = new Document();
-
- int valueCount = (int) counts.getAsLong();
- long[] valueArray = new long[valueCount];
- for (int j = 0; j < valueCount; j++) {
- long value = values.getAsLong();
- valueArray[j] = value;
- doc.add(new SortedNumericDocValuesField("dv", value));
- }
- Arrays.sort(valueArray);
- writeDocValues.add(valueArray);
- for (int j = 0; j < valueCount; j++) {
- doc.add(new StoredField("stored", Long.toString(valueArray[j])));
- }
- writer.addDocument(doc);
- if (random().nextInt(31) == 0) {
- writer.commit();
- }
- }
- writer.forceMerge(1);
-
- writer.close();
-
- // compare
- DirectoryReader ir = DirectoryReader.open(dir);
- TestUtil.checkReader(ir);
- for (LeafReaderContext context : ir.leaves()) {
- LeafReader r = context.reader();
- SortedNumericDocValues docValues = DocValues.getSortedNumeric(r, "dv");
- StoredFields storedFields = r.storedFields();
- for (int i = 0; i < r.maxDoc(); i++) {
- if (i > docValues.docID()) {
- docValues.nextDoc();
- }
- String[] expectedStored = storedFields.document(i).getValues("stored");
- if (i < docValues.docID()) {
- assertEquals(0, expectedStored.length);
- } else {
- long[] readValueArray = new long[docValues.docValueCount()];
- String[] actualDocValue = new String[docValues.docValueCount()];
- for (int j = 0; j < docValues.docValueCount(); ++j) {
- long actualDV = docValues.nextValue();
- readValueArray[j] = actualDV;
- actualDocValue[j] = Long.toString(readValueArray[j]);
- }
- long[] writeValueArray = writeDocValues.get(i);
- // compare write values and read values
- assertArrayEquals(readValueArray, writeValueArray);
-
- // compare dv and stored values
- assertArrayEquals(expectedStored, actualDocValue);
- }
- }
- }
- ir.close();
- dir.close();
- }
-
- private void doTestSparseNumericBlocksOfVariousBitsPerValue(double density) throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
- conf.setMaxBufferedDocs(atLeast(Lucene80DocValuesFormat.NUMERIC_BLOCK_SIZE));
- conf.setRAMBufferSizeMB(-1);
- conf.setMergePolicy(newLogMergePolicy());
- conf.getCodec().compoundFormat().setShouldUseCompoundFile(random().nextBoolean());
- IndexWriter writer = new IndexWriter(dir, conf);
- Document doc = new Document();
- Field storedField = newStringField("stored", "", Field.Store.YES);
- Field dvField = new NumericDocValuesField("dv", 0);
- doc.add(storedField);
- doc.add(dvField);
-
- final int numDocs = atLeast(Lucene80DocValuesFormat.NUMERIC_BLOCK_SIZE * 3);
- final LongSupplier longs = blocksOfVariousBPV();
- for (int i = 0; i < numDocs; i++) {
- if (random().nextDouble() > density) {
- writer.addDocument(new Document());
- continue;
- }
- long value = longs.getAsLong();
- storedField.setStringValue(Long.toString(value));
- dvField.setLongValue(value);
- writer.addDocument(doc);
- }
-
- writer.forceMerge(1);
-
- writer.close();
-
- // compare
- assertDVIterate(dir);
- assertDVAdvance(
- dir, 1); // Tests all jump-lengths from 1 to maxDoc (quite slow ~= 1 minute for 200K docs)
-
- dir.close();
- }
-
- // Tests that advanceExact does not change the outcome
- private void assertDVAdvance(Directory dir, int jumpStep) throws IOException {
- DirectoryReader ir = DirectoryReader.open(dir);
- TestUtil.checkReader(ir);
- for (LeafReaderContext context : ir.leaves()) {
- LeafReader r = context.reader();
- StoredFields storedFields = r.storedFields();
-
- for (int jump = jumpStep; jump < r.maxDoc(); jump += jumpStep) {
- // Create a new instance each time to ensure jumps from the beginning
- NumericDocValues docValues = DocValues.getNumeric(r, "dv");
- for (int docID = 0; docID < r.maxDoc(); docID += jump) {
- String base =
- "document #"
- + docID
- + "/"
- + r.maxDoc()
- + ", jumping "
- + jump
- + " from #"
- + (docID - jump);
- String storedValue = storedFields.document(docID).get("stored");
- if (storedValue == null) {
- assertFalse("There should be no DocValue for " + base, docValues.advanceExact(docID));
- } else {
- assertTrue("There should be a DocValue for " + base, docValues.advanceExact(docID));
- assertEquals(
- "The doc value should be correct for " + base,
- Long.parseLong(storedValue),
- docValues.longValue());
- }
- }
- }
- }
- ir.close();
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/Lucene80NormsConsumer.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/Lucene80NormsConsumer.java
deleted file mode 100644
index aa16a8916ee..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/Lucene80NormsConsumer.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene80;
-
-import static org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat.VERSION_CURRENT;
-
-import java.io.IOException;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.NormsConsumer;
-import org.apache.lucene.codecs.NormsProducer;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.IOUtils;
-
-/** Writer for {@link Lucene80NormsFormat} */
-final class Lucene80NormsConsumer extends NormsConsumer {
- IndexOutput data, meta;
- final int maxDoc;
-
- Lucene80NormsConsumer(
- SegmentWriteState state,
- String dataCodec,
- String dataExtension,
- String metaCodec,
- String metaExtension)
- throws IOException {
- boolean success = false;
- try {
- String dataName =
- IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, dataExtension);
- data = EndiannessReverserUtil.createOutput(state.directory, dataName, state.context);
- CodecUtil.writeIndexHeader(
- data, dataCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
- String metaName =
- IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, metaExtension);
- meta = EndiannessReverserUtil.createOutput(state.directory, metaName, state.context);
- CodecUtil.writeIndexHeader(
- meta, metaCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
- maxDoc = state.segmentInfo.maxDoc();
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(this);
- }
- }
- }
-
- @Override
- public void close() throws IOException {
- boolean success = false;
- try {
- if (meta != null) {
- meta.writeInt(-1); // write EOF marker
- CodecUtil.writeFooter(meta); // write checksum
- }
- if (data != null) {
- CodecUtil.writeFooter(data); // write checksum
- }
- success = true;
- } finally {
- if (success) {
- IOUtils.close(data, meta);
- } else {
- IOUtils.closeWhileHandlingException(data, meta);
- }
- meta = data = null;
- }
- }
-
- @Override
- public void addNormsField(FieldInfo field, NormsProducer normsProducer) throws IOException {
- NumericDocValues values = normsProducer.getNorms(field);
- int numDocsWithValue = 0;
- long min = Long.MAX_VALUE;
- long max = Long.MIN_VALUE;
- for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
- numDocsWithValue++;
- long v = values.longValue();
- min = Math.min(min, v);
- max = Math.max(max, v);
- }
- assert numDocsWithValue <= maxDoc;
-
- meta.writeInt(field.number);
-
- if (numDocsWithValue == 0) {
- meta.writeLong(-2); // docsWithFieldOffset
- meta.writeLong(0L); // docsWithFieldLength
- meta.writeShort((short) -1); // jumpTableEntryCount
- meta.writeByte((byte) -1); // denseRankPower
- } else if (numDocsWithValue == maxDoc) {
- meta.writeLong(-1); // docsWithFieldOffset
- meta.writeLong(0L); // docsWithFieldLength
- meta.writeShort((short) -1); // jumpTableEntryCount
- meta.writeByte((byte) -1); // denseRankPower
- } else {
- long offset = data.getFilePointer();
- meta.writeLong(offset); // docsWithFieldOffset
- values = normsProducer.getNorms(field);
- final short jumpTableEntryCount =
- IndexedDISI.writeBitSet(values, data, IndexedDISI.DEFAULT_DENSE_RANK_POWER);
- meta.writeLong(data.getFilePointer() - offset); // docsWithFieldLength
- meta.writeShort(jumpTableEntryCount);
- meta.writeByte(IndexedDISI.DEFAULT_DENSE_RANK_POWER);
- }
-
- meta.writeInt(numDocsWithValue);
- int numBytesPerValue = numBytesPerValue(min, max);
-
- meta.writeByte((byte) numBytesPerValue);
- if (numBytesPerValue == 0) {
- meta.writeLong(min);
- } else {
- meta.writeLong(data.getFilePointer()); // normsOffset
- values = normsProducer.getNorms(field);
- writeValues(values, numBytesPerValue, data);
- }
- }
-
- private int numBytesPerValue(long min, long max) {
- if (min >= max) {
- return 0;
- } else if (min >= Byte.MIN_VALUE && max <= Byte.MAX_VALUE) {
- return 1;
- } else if (min >= Short.MIN_VALUE && max <= Short.MAX_VALUE) {
- return 2;
- } else if (min >= Integer.MIN_VALUE && max <= Integer.MAX_VALUE) {
- return 4;
- } else {
- return 8;
- }
- }
-
- private void writeValues(NumericDocValues values, int numBytesPerValue, IndexOutput out)
- throws IOException, AssertionError {
- for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
- long value = values.longValue();
- switch (numBytesPerValue) {
- case 1:
- out.writeByte((byte) value);
- break;
- case 2:
- out.writeShort((short) value);
- break;
- case 4:
- out.writeInt((int) value);
- break;
- case 8:
- out.writeLong(value);
- break;
- default:
- throw new AssertionError();
- }
- }
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/Lucene80RWNormsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/Lucene80RWNormsFormat.java
deleted file mode 100644
index 1eee0f256f9..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/Lucene80RWNormsFormat.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene80;
-
-import java.io.IOException;
-import org.apache.lucene.codecs.NormsConsumer;
-import org.apache.lucene.index.SegmentWriteState;
-
-public class Lucene80RWNormsFormat extends Lucene80NormsFormat {
-
- @Override
- public NormsConsumer normsConsumer(SegmentWriteState state) throws IOException {
- return new Lucene80NormsConsumer(
- state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestBestCompressionLucene80DocValuesFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestBestCompressionLucene80DocValuesFormat.java
deleted file mode 100644
index 93ca2fe3cee..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestBestCompressionLucene80DocValuesFormat.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene80;
-
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-import org.apache.lucene.tests.util.TestUtil;
-
-/** Tests Lucene80DocValuesFormat */
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestBestCompressionLucene80DocValuesFormat
- extends BaseLucene80DocValuesFormatTestCase {
- private final Codec codec =
- TestUtil.alwaysDocValuesFormat(
- new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION));
-
- @Override
- protected Codec getCodec() {
- return codec;
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestBestSpeedLucene80DocValuesFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestBestSpeedLucene80DocValuesFormat.java
deleted file mode 100644
index 2bdb660ca70..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestBestSpeedLucene80DocValuesFormat.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene80;
-
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-import org.apache.lucene.tests.util.TestUtil;
-
-/** Tests Lucene80DocValuesFormat */
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestBestSpeedLucene80DocValuesFormat extends BaseLucene80DocValuesFormatTestCase {
- private final Codec codec =
- TestUtil.alwaysDocValuesFormat(
- new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_SPEED));
-
- @Override
- protected Codec getCodec() {
- return codec;
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestDocValuesCompression.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestDocValuesCompression.java
deleted file mode 100644
index b1ab62c3595..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestDocValuesCompression.java
+++ /dev/null
@@ -1,320 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene80;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.SortedDocValuesField;
-import org.apache.lucene.document.SortedSetDocValuesField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-import org.apache.lucene.tests.util.TestUtil;
-import org.apache.lucene.util.BytesRef;
-
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestDocValuesCompression extends LuceneTestCase {
- private final Codec bestSpeed =
- TestUtil.alwaysDocValuesFormat(
- new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_SPEED));
-
- private final Codec bestCompression =
- TestUtil.alwaysDocValuesFormat(
- new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION));
-
- public void testTermsDictCompressionForLowCardinalityFields() throws IOException {
- final int CARDINALITY = Lucene80DocValuesFormat.TERMS_DICT_BLOCK_COMPRESSION_THRESHOLD - 1;
- Set<String> valuesSet = new HashSet<>();
- for (int i = 0; i < CARDINALITY; ++i) {
- final int length = TestUtil.nextInt(random(), 10, 30);
- String value = TestUtil.randomSimpleString(random(), length);
- valuesSet.add(value);
- }
-
- List<String> values = new ArrayList<>(valuesSet);
- long sizeForBestSpeed = writeAndGetDocValueFileSize(bestSpeed, values);
- long sizeForBestCompression = writeAndGetDocValueFileSize(bestCompression, values);
-
- // Ensure terms dict data was not compressed for low-cardinality fields.
- assertEquals(sizeForBestSpeed, sizeForBestCompression);
- }
-
- public void testTermsDictCompressionForHighCardinalityFields() throws IOException {
- final int CARDINALITY = Lucene80DocValuesFormat.TERMS_DICT_BLOCK_COMPRESSION_THRESHOLD << 1;
- Set<String> valuesSet = new HashSet<>();
- for (int i = 0; i < CARDINALITY; ++i) {
- final int length = TestUtil.nextInt(random(), 10, 30);
- String value = TestUtil.randomSimpleString(random(), length);
- // Add common suffix for better compression ratio.
- valuesSet.add(value + "_CommonPartBetterForCompression");
- }
-
- List<String> values = new ArrayList<>(valuesSet);
- long sizeForBestSpeed = writeAndGetDocValueFileSize(bestSpeed, values);
- long sizeForBestCompression = writeAndGetDocValueFileSize(bestCompression, values);
-
- // Compression happened.
- assertTrue(sizeForBestCompression < sizeForBestSpeed);
- }
-
- public void testReseekAfterSkipDecompression() throws IOException {
- final int CARDINALITY = (Lucene80DocValuesFormat.TERMS_DICT_BLOCK_LZ4_SIZE << 1) + 11;
- Set<String> valueSet = HashSet.newHashSet(CARDINALITY);
- for (int i = 0; i < CARDINALITY; i++) {
- valueSet.add(TestUtil.randomSimpleString(random(), 64));
- }
- List<String> values = new ArrayList<>(valueSet);
- Collections.sort(values);
- // Create one non-existent value just between block-1 and block-2.
- String nonexistentValue =
- values.get(Lucene80DocValuesFormat.TERMS_DICT_BLOCK_LZ4_SIZE - 1)
- + TestUtil.randomSimpleString(random(), 64, 128);
- int docValues = values.size();
-
- try (Directory directory = newDirectory()) {
- Analyzer analyzer = new StandardAnalyzer();
- IndexWriterConfig config = new IndexWriterConfig(analyzer);
- config.setCodec(bestCompression);
- config.setUseCompoundFile(false);
- IndexWriter writer = new IndexWriter(directory, config);
- for (int i = 0; i < 280; i++) {
- Document doc = new Document();
- doc.add(new StringField("id", "Doc" + i, Field.Store.NO));
- doc.add(new SortedDocValuesField("sdv", new BytesRef(values.get(i % docValues))));
- writer.addDocument(doc);
- }
- writer.commit();
- writer.forceMerge(1);
- DirectoryReader dReader = DirectoryReader.open(writer);
- writer.close();
-
- LeafReader reader = getOnlyLeafReader(dReader);
- // Check values count.
- SortedDocValues ssdvMulti = reader.getSortedDocValues("sdv");
- assertEquals(docValues, ssdvMulti.getValueCount());
-
- // Seek to first block.
- int ord1 = ssdvMulti.lookupTerm(new BytesRef(values.get(0)));
- assertTrue(ord1 >= 0);
- int ord2 = ssdvMulti.lookupTerm(new BytesRef(values.get(1)));
- assertTrue(ord2 >= ord1);
- // Ensure re-seek logic is correct after skip-decompression.
- int nonexistentOrd2 = ssdvMulti.lookupTerm(new BytesRef(nonexistentValue));
- assertTrue(nonexistentOrd2 < 0);
- dReader.close();
- }
- }
-
- public void testLargeTermsCompression() throws IOException {
- final int CARDINALITY = Lucene80DocValuesFormat.TERMS_DICT_BLOCK_COMPRESSION_THRESHOLD << 1;
- Set<String> valuesSet = new HashSet<>();
- for (int i = 0; i < CARDINALITY; ++i) {
- final int length = TestUtil.nextInt(random(), 512, 1024);
- valuesSet.add(TestUtil.randomSimpleString(random(), length));
- }
- int valuesCount = valuesSet.size();
- List<String> values = new ArrayList<>(valuesSet);
-
- try (Directory directory = newDirectory()) {
- Analyzer analyzer = new StandardAnalyzer();
- IndexWriterConfig config = new IndexWriterConfig(analyzer);
- config.setCodec(bestCompression);
- config.setUseCompoundFile(false);
- IndexWriter writer = new IndexWriter(directory, config);
- for (int i = 0; i < 256; i++) {
- Document doc = new Document();
- doc.add(new StringField("id", "Doc" + i, Field.Store.NO));
- doc.add(new SortedDocValuesField("sdv", new BytesRef(values.get(i % valuesCount))));
- writer.addDocument(doc);
- }
- writer.commit();
- writer.forceMerge(1);
- DirectoryReader ireader = DirectoryReader.open(writer);
- writer.close();
-
- LeafReader reader = getOnlyLeafReader(ireader);
- // Check values count.
- SortedDocValues ssdvMulti = reader.getSortedDocValues("sdv");
- assertEquals(valuesCount, ssdvMulti.getValueCount());
- ireader.close();
- }
- }
-
- // Ensure the old segment can be merged together with the new compressed segment.
- public void testMergeWithUncompressedSegment() throws IOException {
- final int CARDINALITY = Lucene80DocValuesFormat.TERMS_DICT_BLOCK_COMPRESSION_THRESHOLD << 1;
- Set<String> valuesSet = new HashSet<>();
- for (int i = 0; i < CARDINALITY; ++i) {
- final int length = TestUtil.nextInt(random(), 10, 30);
- // Add common suffix for better compression ratio.
- valuesSet.add(TestUtil.randomSimpleString(random(), length));
- }
- List<String> values = new ArrayList<>(valuesSet);
- int valuesCount = values.size();
-
- try (Directory directory = newDirectory()) {
- // 1. Write 256 documents without terms dict compression.
- Analyzer analyzer = new StandardAnalyzer();
- IndexWriterConfig config = new IndexWriterConfig(analyzer);
- config.setCodec(bestSpeed);
- config.setUseCompoundFile(false);
- IndexWriter writer = new IndexWriter(directory, config);
- for (int i = 0; i < 256; i++) {
- Document doc = new Document();
- doc.add(new StringField("id", "Doc" + i, Field.Store.NO));
- doc.add(new SortedSetDocValuesField("ssdv", new BytesRef(values.get(i % valuesCount))));
- doc.add(
- new SortedSetDocValuesField("ssdv", new BytesRef(values.get((i + 1) % valuesCount))));
- doc.add(
- new SortedSetDocValuesField("ssdv", new BytesRef(values.get((i + 2) % valuesCount))));
- doc.add(new SortedDocValuesField("sdv", new BytesRef(values.get(i % valuesCount))));
- writer.addDocument(doc);
- }
- writer.commit();
- DirectoryReader ireader = DirectoryReader.open(writer);
- assertEquals(256, ireader.numDocs());
- LeafReader reader = getOnlyLeafReader(ireader);
- SortedSetDocValues ssdv = reader.getSortedSetDocValues("ssdv");
- assertEquals(valuesCount, ssdv.getValueCount());
- SortedDocValues sdv = reader.getSortedDocValues("sdv");
- assertEquals(valuesCount, sdv.getValueCount());
- ireader.close();
- writer.close();
-
- // 2. Add another 100 documents, and enabling terms dict compression.
- config = new IndexWriterConfig(analyzer);
- config.setCodec(bestCompression);
- config.setUseCompoundFile(false);
- writer = new IndexWriter(directory, config);
- // Add 2 new values.
- valuesSet.add(TestUtil.randomSimpleString(random(), 10));
- valuesSet.add(TestUtil.randomSimpleString(random(), 10));
- values = new ArrayList<>(valuesSet);
- valuesCount = valuesSet.size();
-
- for (int i = 256; i < 356; i++) {
- Document doc = new Document();
- doc.add(new StringField("id", "Doc" + i, Field.Store.NO));
- doc.add(new SortedSetDocValuesField("ssdv", new BytesRef(values.get(i % valuesCount))));
- doc.add(new SortedDocValuesField("sdv", new BytesRef(values.get(i % valuesCount))));
- writer.addDocument(doc);
- }
- writer.commit();
- writer.forceMerge(1);
- ireader = DirectoryReader.open(writer);
- assertEquals(356, ireader.numDocs());
- reader = getOnlyLeafReader(ireader);
- ssdv = reader.getSortedSetDocValues("ssdv");
- assertEquals(valuesCount, ssdv.getValueCount());
- ireader.close();
- writer.close();
- }
- }
-
- private static long writeAndGetDocValueFileSize(Codec codec, List<String> values)
- throws IOException {
- int valuesCount = values.size();
- long dvdFileSize = -1;
- try (Directory directory = newDirectory()) {
- Analyzer analyzer = new StandardAnalyzer();
- IndexWriterConfig config = new IndexWriterConfig(analyzer);
- config.setCodec(codec);
- config.setUseCompoundFile(false);
- IndexWriter writer = new IndexWriter(directory, config);
- for (int i = 0; i < 256; i++) {
- Document doc = new Document();
- doc.add(new StringField("id", "Doc" + i, Field.Store.NO));
- // Multi value sorted-set field.
- doc.add(
- new SortedSetDocValuesField("ssdv_multi_", new BytesRef(values.get(i % valuesCount))));
- doc.add(
- new SortedSetDocValuesField(
- "ssdv_multi_", new BytesRef(values.get((i + 1) % valuesCount))));
- doc.add(
- new SortedSetDocValuesField(
- "ssdv_multi_", new BytesRef(values.get((i + 2) % valuesCount))));
- // Single value sorted-set field.
- doc.add(
- new SortedSetDocValuesField("ssdv_single_", new BytesRef(values.get(i % valuesCount))));
- // Sorted field.
- doc.add(new SortedDocValuesField("sdv", new BytesRef(values.get(i % valuesCount))));
- writer.addDocument(doc);
- }
- writer.commit();
- writer.forceMerge(1);
- DirectoryReader ireader = DirectoryReader.open(writer);
- writer.close();
-
- LeafReader reader = getOnlyLeafReader(ireader);
- // Check values count.
- SortedSetDocValues ssdvMulti = reader.getSortedSetDocValues("ssdv_multi_");
- assertEquals(valuesCount, ssdvMulti.getValueCount());
- for (int i = 0; i < valuesCount; i++) {
- BytesRef term = ssdvMulti.lookupOrd(i);
- assertTrue(term.bytes.length > 0);
- }
- for (int i = 0; i < valuesCount; i++) {
- for (int j = 0; j < 3; j++) {
- assertTrue(ssdvMulti.lookupTerm(new BytesRef(values.get((i + j) % valuesCount))) >= 0);
- }
- }
-
- SortedSetDocValues ssdvSingle = reader.getSortedSetDocValues("ssdv_single_");
- assertEquals(valuesCount, ssdvSingle.getValueCount());
- for (int i = 0; i < valuesCount; i++) {
- assertTrue(ssdvSingle.lookupTerm(new BytesRef(values.get(i % valuesCount))) >= 0);
- }
-
- SortedDocValues sdv = reader.getSortedDocValues("sdv");
- assertEquals(valuesCount, sdv.getValueCount());
- for (int i = 0; i < valuesCount; i++) {
- assertTrue(sdv.lookupTerm(new BytesRef(values.get(i % valuesCount))) >= 0);
- }
-
- dvdFileSize = docValueFileSize(directory);
- assertTrue(dvdFileSize > 0);
- ireader.close();
- }
-
- return dvdFileSize;
- }
-
- static long docValueFileSize(Directory d) throws IOException {
- for (String file : d.listAll()) {
- if (file.endsWith(Lucene80DocValuesFormat.DATA_EXTENSION)) {
- return d.fileLength(file);
- }
- }
- return -1;
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestIndexedDISI.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestIndexedDISI.java
deleted file mode 100644
index 800c31d5e5d..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestIndexedDISI.java
+++ /dev/null
@@ -1,588 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene80;
-
-import java.io.IOException;
-import java.util.Random;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.RandomAccessInput;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-import org.apache.lucene.tests.util.TestUtil;
-import org.apache.lucene.util.BitSet;
-import org.apache.lucene.util.BitSetIterator;
-import org.apache.lucene.util.FixedBitSet;
-import org.apache.lucene.util.SparseFixedBitSet;
-
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestIndexedDISI extends LuceneTestCase {
-
- public void testEmpty() throws IOException {
- int maxDoc = TestUtil.nextInt(random(), 1, 100000);
- BitSet set = new SparseFixedBitSet(maxDoc);
- try (Directory dir = newDirectory()) {
- doTest(set, dir);
- }
- }
-
- // EMPTY blocks are special with regard to jumps as they have size 0
- @Nightly
- public void testEmptyBlocks() throws IOException {
- final int B = 65536;
- int maxDoc = B * 11;
- BitSet set = new SparseFixedBitSet(maxDoc);
- // block 0: EMPTY
- set.set(B + 5); // block 1: SPARSE
- // block 2: EMPTY
- // block 3: EMPTY
- set.set(B * 4 + 5); // block 4: SPARSE
-
- for (int i = 0; i < B; i++) {
- set.set(B * 6 + i); // block 6: ALL
- }
- for (int i = 0; i < B; i += 3) {
- set.set(B * 7 + i); // block 7: DENSE
- }
- for (int i = 0; i < B; i++) {
- if (i != 32768) {
- set.set(B * 8 + i); // block 8: DENSE (all-1)
- }
- }
- // block 9-11: EMPTY
-
- try (Directory dir = newDirectory()) {
- doTestAllSingleJump(set, dir);
- }
-
- // Change the first block to DENSE to see if jump-tables sets to position 0
- set.set(0);
- try (Directory dir = newDirectory()) {
- doTestAllSingleJump(set, dir);
- }
- }
-
- // EMPTY blocks are special with regard to jumps as they have size 0
- public void testLastEmptyBlocks() throws IOException {
- final int B = 65536;
- int maxDoc = B * 3;
- BitSet set = new SparseFixedBitSet(maxDoc);
- for (int docID = 0; docID < B * 2; docID++) { // first 2 blocks are ALL
- set.set(docID);
- }
- // Last block is EMPTY
-
- try (Directory dir = newDirectory()) {
- doTestAllSingleJump(set, dir);
- assertAdvanceBeyondEnd(set, dir);
- }
- }
-
- // Checks that advance after the end of the blocks has been reached has the correct behaviour
- private void assertAdvanceBeyondEnd(BitSet set, Directory dir) throws IOException {
- final int cardinality = set.cardinality();
- final byte denseRankPower = 9; // Not tested here so fixed to isolate factors
- int jumpTableentryCount;
- try (IndexOutput out = EndiannessReverserUtil.createOutput(dir, "bar", IOContext.DEFAULT)) {
- jumpTableentryCount =
- IndexedDISI.writeBitSet(new BitSetIterator(set, cardinality), out, denseRankPower);
- }
-
- try (IndexInput in = EndiannessReverserUtil.openInput(dir, "bar", IOContext.DEFAULT)) {
- BitSetIterator disi2 = new BitSetIterator(set, cardinality);
- int doc = disi2.docID();
- int index = 0;
- while (doc < cardinality) {
- doc = disi2.nextDoc();
- index++;
- }
-
- IndexedDISI disi =
- new IndexedDISI(in, 0L, in.length(), jumpTableentryCount, denseRankPower, cardinality);
- // Advance 1 docID beyond end
- assertFalse(
- "There should be no set bit beyond the valid docID range",
- disi.advanceExact(set.length()));
- disi.advance(
- doc); // Should be the special docID signifyin NO_MORE_DOCS from the BitSetIterator
- assertEquals(
- "The index when advancing beyond the last defined docID should be correct",
- index,
- disi.index() + 1); // disi.index()+1 as the while-loop also counts the NO_MORE_DOCS
- }
- }
-
- // TODO: can this be toned down?
- @Nightly
- public void testRandomBlocks() throws IOException {
- final int BLOCKS = 5;
- BitSet set = createSetWithRandomBlocks(BLOCKS);
- try (Directory dir = newDirectory()) {
- doTestAllSingleJump(set, dir);
- }
- }
-
- // When doing merges in Lucene80NormsProducer, IndexedDISI are created from slices where the
- // offset is not 0
- public void testPositionNotZero() throws IOException {
- final int BLOCKS = 10;
- final byte denseRankPower =
- rarely() ? -1 : (byte) (random().nextInt(7) + 7); // sane + chance of disable
-
- BitSet set = createSetWithRandomBlocks(BLOCKS);
- try (Directory dir = newDirectory()) {
- final int cardinality = set.cardinality();
- int jumpTableEntryCount;
- try (IndexOutput out = EndiannessReverserUtil.createOutput(dir, "foo", IOContext.DEFAULT)) {
- jumpTableEntryCount =
- IndexedDISI.writeBitSet(new BitSetIterator(set, cardinality), out, denseRankPower);
- }
- try (IndexInput fullInput = EndiannessReverserUtil.openInput(dir, "foo", IOContext.DEFAULT)) {
- IndexInput blockData =
- IndexedDISI.createBlockSlice(
- fullInput, "blocks", 0, fullInput.length(), jumpTableEntryCount);
- blockData.seek(random().nextInt((int) blockData.length()));
-
- RandomAccessInput jumpTable =
- IndexedDISI.createJumpTable(fullInput, 0, fullInput.length(), jumpTableEntryCount);
- IndexedDISI disi =
- new IndexedDISI(blockData, jumpTable, jumpTableEntryCount, denseRankPower, cardinality);
- // This failed at some point during LUCENE-8585 development as it did not reset the slice
- // position
- disi.advanceExact(BLOCKS * 65536 - 1);
- }
- }
- }
-
- private BitSet createSetWithRandomBlocks(int blockCount) {
- final int B = 65536;
- BitSet set = new SparseFixedBitSet(blockCount * B);
- for (int block = 0; block < blockCount; block++) {
- switch (random().nextInt(4)) {
- case 0:
- { // EMPTY
- break;
- }
- case 1:
- { // ALL
- for (int docID = block * B; docID < (block + 1) * B; docID++) {
- set.set(docID);
- }
- break;
- }
- case 2:
- { // SPARSE ( < 4096 )
- for (int docID = block * B; docID < (block + 1) * B; docID += 101) {
- set.set(docID);
- }
- break;
- }
- case 3:
- { // DENSE ( >= 4096 )
- for (int docID = block * B; docID < (block + 1) * B; docID += 3) {
- set.set(docID);
- }
- break;
- }
- default:
- throw new IllegalStateException(
- "Modulo logic error: there should only be 4 possibilities");
- }
- }
- return set;
- }
-
- private void doTestAllSingleJump(BitSet set, Directory dir) throws IOException {
- final int cardinality = set.cardinality();
- final byte denseRankPower =
- rarely() ? -1 : (byte) (random().nextInt(7) + 7); // sane + chance of disable
- long length;
- int jumpTableentryCount;
- try (IndexOutput out = EndiannessReverserUtil.createOutput(dir, "foo", IOContext.DEFAULT)) {
- jumpTableentryCount =
- IndexedDISI.writeBitSet(new BitSetIterator(set, cardinality), out, denseRankPower);
- length = out.getFilePointer();
- }
-
- try (IndexInput in = EndiannessReverserUtil.openInput(dir, "foo", IOContext.DEFAULT)) {
- for (int i = 0; i < set.length(); i++) {
- IndexedDISI disi =
- new IndexedDISI(in, 0L, length, jumpTableentryCount, denseRankPower, cardinality);
- assertEquals(
- "The bit at " + i + " should be correct with advanceExact",
- set.get(i),
- disi.advanceExact(i));
-
- IndexedDISI disi2 =
- new IndexedDISI(in, 0L, length, jumpTableentryCount, denseRankPower, cardinality);
- disi2.advance(i);
- // Proper sanity check with jump tables as an error could make them seek backwards
- assertTrue(
- "The docID should at least be "
- + i
- + " after advance("
- + i
- + ") but was "
- + disi2.docID(),
- i <= disi2.docID());
- if (set.get(i)) {
- assertEquals("The docID should be present with advance", i, disi2.docID());
- } else {
- assertNotSame("The docID should not be present with advance", i, disi2.docID());
- }
- }
- }
- }
-
- public void testOneDoc() throws IOException {
- int maxDoc = TestUtil.nextInt(random(), 1, 100000);
- BitSet set = new SparseFixedBitSet(maxDoc);
- set.set(random().nextInt(maxDoc));
- try (Directory dir = newDirectory()) {
- doTest(set, dir);
- }
- }
-
- public void testTwoDocs() throws IOException {
- int maxDoc = TestUtil.nextInt(random(), 1, 100000);
- BitSet set = new SparseFixedBitSet(maxDoc);
- set.set(random().nextInt(maxDoc));
- set.set(random().nextInt(maxDoc));
- try (Directory dir = newDirectory()) {
- doTest(set, dir);
- }
- }
-
- public void testAllDocs() throws IOException {
- int maxDoc = TestUtil.nextInt(random(), 1, 100000);
- FixedBitSet set = new FixedBitSet(maxDoc);
- set.set(1, maxDoc);
- try (Directory dir = newDirectory()) {
- doTest(set, dir);
- }
- }
-
- public void testHalfFull() throws IOException {
- int maxDoc = TestUtil.nextInt(random(), 1, 100000);
- BitSet set = new SparseFixedBitSet(maxDoc);
- for (int i = random().nextInt(2); i < maxDoc; i += TestUtil.nextInt(random(), 1, 3)) {
- set.set(i);
- }
- try (Directory dir = newDirectory()) {
- doTest(set, dir);
- }
- }
-
- public void testDocRange() throws IOException {
- try (Directory dir = newDirectory()) {
- for (int iter = 0; iter < 10; ++iter) {
- int maxDoc = TestUtil.nextInt(random(), 1, 1000000);
- FixedBitSet set = new FixedBitSet(maxDoc);
- final int start = random().nextInt(maxDoc);
- final int end = TestUtil.nextInt(random(), start + 1, maxDoc);
- set.set(start, end);
- doTest(set, dir);
- }
- }
- }
-
- public void testSparseDenseBoundary() throws IOException {
- try (Directory dir = newDirectory()) {
- FixedBitSet set = new FixedBitSet(200000);
- int start = 65536 + random().nextInt(100);
- final byte denseRankPower =
- rarely() ? -1 : (byte) (random().nextInt(7) + 7); // sane + chance of disable
-
- // we set MAX_ARRAY_LENGTH bits so the encoding will be sparse
- set.set(start, start + IndexedDISI.MAX_ARRAY_LENGTH);
- long length;
- int jumpTableEntryCount;
- try (IndexOutput out =
- EndiannessReverserUtil.createOutput(dir, "sparse", IOContext.DEFAULT)) {
- jumpTableEntryCount =
- IndexedDISI.writeBitSet(
- new BitSetIterator(set, IndexedDISI.MAX_ARRAY_LENGTH), out, denseRankPower);
- length = out.getFilePointer();
- }
- try (IndexInput in = EndiannessReverserUtil.openInput(dir, "sparse", IOContext.DEFAULT)) {
- IndexedDISI disi =
- new IndexedDISI(
- in, 0L, length, jumpTableEntryCount, denseRankPower, IndexedDISI.MAX_ARRAY_LENGTH);
- assertEquals(start, disi.nextDoc());
- assertEquals(IndexedDISI.Method.SPARSE, disi.method);
- }
- doTest(set, dir);
-
- // now we set one more bit so the encoding will be dense
- set.set(start + IndexedDISI.MAX_ARRAY_LENGTH + random().nextInt(100));
- try (IndexOutput out = EndiannessReverserUtil.createOutput(dir, "bar", IOContext.DEFAULT)) {
- IndexedDISI.writeBitSet(
- new BitSetIterator(set, IndexedDISI.MAX_ARRAY_LENGTH + 1), out, denseRankPower);
- length = out.getFilePointer();
- }
- try (IndexInput in = EndiannessReverserUtil.openInput(dir, "bar", IOContext.DEFAULT)) {
- IndexedDISI disi =
- new IndexedDISI(
- in,
- 0L,
- length,
- jumpTableEntryCount,
- denseRankPower,
- IndexedDISI.MAX_ARRAY_LENGTH + 1);
- assertEquals(start, disi.nextDoc());
- assertEquals(IndexedDISI.Method.DENSE, disi.method);
- }
- doTest(set, dir);
- }
- }
-
- public void testOneDocMissing() throws IOException {
- int maxDoc = TestUtil.nextInt(random(), 1, 1000000);
- FixedBitSet set = new FixedBitSet(maxDoc);
- set.set(0, maxDoc);
- set.clear(random().nextInt(maxDoc));
- try (Directory dir = newDirectory()) {
- doTest(set, dir);
- }
- }
-
- public void testFewMissingDocs() throws IOException {
- try (Directory dir = newDirectory()) {
- int numIters = atLeast(10);
- for (int iter = 0; iter < numIters; ++iter) {
- int maxDoc = TestUtil.nextInt(random(), 1, 100000);
- FixedBitSet set = new FixedBitSet(maxDoc);
- set.set(0, maxDoc);
- final int numMissingDocs = TestUtil.nextInt(random(), 2, 1000);
- for (int i = 0; i < numMissingDocs; ++i) {
- set.clear(random().nextInt(maxDoc));
- }
- doTest(set, dir);
- }
- }
- }
-
- public void testDenseMultiBlock() throws IOException {
- try (Directory dir = newDirectory()) {
- int maxDoc = 10 * 65536; // 10 blocks
- FixedBitSet set = new FixedBitSet(maxDoc);
- for (int i = 0; i < maxDoc; i += 2) { // Set every other to ensure dense
- set.set(i);
- }
- doTest(set, dir);
- }
- }
-
- public void testIllegalDenseRankPower() throws IOException {
-
- // Legal values
- for (byte denseRankPower : new byte[] {-1, 7, 8, 9, 10, 11, 12, 13, 14, 15}) {
- createAndOpenDISI(denseRankPower, denseRankPower);
- }
-
- // Illegal values
- for (byte denseRankPower : new byte[] {-2, 0, 1, 6, 16}) {
- expectThrows(
- IllegalArgumentException.class,
- () -> {
- createAndOpenDISI(
- denseRankPower, (byte) 8); // Illegal write, legal read (should not reach read)
- });
-
- expectThrows(
- IllegalArgumentException.class,
- () -> {
- createAndOpenDISI(
- (byte) 8, denseRankPower); // Legal write, illegal read (should reach read)
- });
- }
- }
-
- private void createAndOpenDISI(byte denseRankPowerWrite, byte denseRankPowerRead)
- throws IOException {
- BitSet set = new FixedBitSet(10);
- set.set(set.length() - 1);
- try (Directory dir = newDirectory()) {
- long length;
- int jumpTableEntryCount = -1;
- try (IndexOutput out = EndiannessReverserUtil.createOutput(dir, "foo", IOContext.DEFAULT)) {
- jumpTableEntryCount =
- IndexedDISI.writeBitSet(
- new BitSetIterator(set, set.cardinality()), out, denseRankPowerWrite);
- length = out.getFilePointer();
- }
- try (IndexInput in = EndiannessReverserUtil.openInput(dir, "foo", IOContext.DEFAULT)) {
- new IndexedDISI(in, 0L, length, jumpTableEntryCount, denseRankPowerRead, set.cardinality());
- }
- // This tests the legality of the denseRankPower only, so we don't do anything with the disi
- }
- }
-
- public void testOneDocMissingFixed() throws IOException {
- int maxDoc = 9699;
- final byte denseRankPower =
- rarely() ? -1 : (byte) (random().nextInt(7) + 7); // sane + chance of disable
- FixedBitSet set = new FixedBitSet(maxDoc);
- set.set(0, maxDoc);
- set.clear(1345);
- try (Directory dir = newDirectory()) {
-
- final int cardinality = set.cardinality();
- long length;
- int jumpTableentryCount;
- try (IndexOutput out = EndiannessReverserUtil.createOutput(dir, "foo", IOContext.DEFAULT)) {
- jumpTableentryCount =
- IndexedDISI.writeBitSet(new BitSetIterator(set, cardinality), out, denseRankPower);
- length = out.getFilePointer();
- }
-
- int step = 16000;
- try (IndexInput in = EndiannessReverserUtil.openInput(dir, "foo", IOContext.DEFAULT)) {
- IndexedDISI disi =
- new IndexedDISI(in, 0L, length, jumpTableentryCount, denseRankPower, cardinality);
- BitSetIterator disi2 = new BitSetIterator(set, cardinality);
- assertAdvanceEquality(disi, disi2, step);
- }
- }
- }
-
- public void testRandom() throws IOException {
- try (Directory dir = newDirectory()) {
- int numIters = atLeast(3);
- for (int i = 0; i < numIters; ++i) {
- doTestRandom(dir);
- }
- }
- }
-
- private void doTestRandom(Directory dir) throws IOException {
- Random random = random();
- final int maxStep = TestUtil.nextInt(random, 1, 1 << TestUtil.nextInt(random, 2, 20));
- final int numDocs =
- TestUtil.nextInt(random, 1, Math.min(100000, (Integer.MAX_VALUE - 1) / maxStep));
- BitSet docs = new SparseFixedBitSet(numDocs * maxStep + 1);
- int lastDoc = -1;
- for (int doc = -1, i = 0; i < numDocs; ++i) {
- doc += TestUtil.nextInt(random, 1, maxStep);
- docs.set(doc);
- lastDoc = doc;
- }
- final int maxDoc = lastDoc + TestUtil.nextInt(random, 1, 100);
-
- BitSet set = BitSet.of(new BitSetIterator(docs, docs.approximateCardinality()), maxDoc);
- doTest(set, dir);
- }
-
- private void doTest(BitSet set, Directory dir) throws IOException {
- final int cardinality = set.cardinality();
- final byte denseRankPower =
- rarely() ? -1 : (byte) (random().nextInt(7) + 7); // sane + chance of disable
- long length;
- int jumpTableentryCount;
- try (IndexOutput out = EndiannessReverserUtil.createOutput(dir, "foo", IOContext.DEFAULT)) {
- jumpTableentryCount =
- IndexedDISI.writeBitSet(new BitSetIterator(set, cardinality), out, denseRankPower);
- length = out.getFilePointer();
- }
-
- try (IndexInput in = EndiannessReverserUtil.openInput(dir, "foo", IOContext.DEFAULT)) {
- IndexedDISI disi =
- new IndexedDISI(in, 0L, length, jumpTableentryCount, denseRankPower, cardinality);
- BitSetIterator disi2 = new BitSetIterator(set, cardinality);
- assertSingleStepEquality(disi, disi2);
- }
-
- for (int step : new int[] {1, 10, 100, 1000, 10000, 100000}) {
- try (IndexInput in = EndiannessReverserUtil.openInput(dir, "foo", IOContext.DEFAULT)) {
- IndexedDISI disi =
- new IndexedDISI(in, 0L, length, jumpTableentryCount, denseRankPower, cardinality);
- BitSetIterator disi2 = new BitSetIterator(set, cardinality);
- assertAdvanceEquality(disi, disi2, step);
- }
- }
-
- for (int step : new int[] {10, 100, 1000, 10000, 100000}) {
- try (IndexInput in = EndiannessReverserUtil.openInput(dir, "foo", IOContext.DEFAULT)) {
- IndexedDISI disi =
- new IndexedDISI(in, 0L, length, jumpTableentryCount, denseRankPower, cardinality);
- BitSetIterator disi2 = new BitSetIterator(set, cardinality);
- int disi2length = set.length();
- assertAdvanceExactRandomized(disi, disi2, disi2length, step);
- }
- }
-
- dir.deleteFile("foo");
- }
-
- private void assertAdvanceExactRandomized(
- IndexedDISI disi, BitSetIterator disi2, int disi2length, int step) throws IOException {
- int index = -1;
- Random random = random();
- for (int target = 0; target < disi2length; ) {
- target += TestUtil.nextInt(random, 0, step);
- int doc = disi2.docID();
- while (doc < target) {
- doc = disi2.nextDoc();
- index++;
- }
-
- boolean exists = disi.advanceExact(target);
- assertEquals(doc == target, exists);
- if (exists) {
- assertEquals(index, disi.index());
- } else if (random.nextBoolean()) {
- assertEquals(doc, disi.nextDoc());
- // This is a bit strange when doc == NO_MORE_DOCS as the index overcounts in the disi2
- // while-loop
- assertEquals(index, disi.index());
- target = doc;
- }
- }
- }
-
- private void assertSingleStepEquality(IndexedDISI disi, BitSetIterator disi2) throws IOException {
- int i = 0;
- for (int doc = disi2.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = disi2.nextDoc()) {
- assertEquals(doc, disi.nextDoc());
- assertEquals(i++, disi.index());
- }
- assertEquals(DocIdSetIterator.NO_MORE_DOCS, disi.nextDoc());
- }
-
- private void assertAdvanceEquality(IndexedDISI disi, BitSetIterator disi2, int step)
- throws IOException {
- int index = -1;
- while (true) {
- int target = disi2.docID() + step;
- int doc;
- do {
- doc = disi2.nextDoc();
- index++;
- } while (doc < target);
- assertEquals(doc, disi.advance(target));
- if (doc == DocIdSetIterator.NO_MORE_DOCS) {
- break;
- }
- assertEquals(
- "Expected equality using step " + step + " at docID " + doc, index, disi.index());
- }
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestLucene80NormsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestLucene80NormsFormat.java
deleted file mode 100644
index c61fb2beae4..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestLucene80NormsFormat.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene80;
-
-import org.apache.lucene.backward_codecs.lucene87.Lucene87RWCodec;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.tests.index.BaseNormsFormatTestCase;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-
-/** Tests Lucene80NormsFormat */
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestLucene80NormsFormat extends BaseNormsFormatTestCase {
- private final Codec codec = new Lucene87RWCodec();
-
- @Override
- protected Codec getCodec() {
- return codec;
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestLucene80NormsFormatMergeInstance.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestLucene80NormsFormatMergeInstance.java
deleted file mode 100644
index 851d8f82056..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/TestLucene80NormsFormatMergeInstance.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene80;
-
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-
-/** Test the merge instance of the Lucene80 norms format. */
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestLucene80NormsFormatMergeInstance extends TestLucene80NormsFormat {
-
- @Override
- protected boolean shouldTestMergeInstance() {
- return true;
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84PostingsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84PostingsWriter.java
deleted file mode 100644
index 3fa503c033e..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84PostingsWriter.java
+++ /dev/null
@@ -1,549 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene84;
-
-import static org.apache.lucene.backward_codecs.lucene84.ForUtil.BLOCK_SIZE;
-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.DOC_CODEC;
-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.MAX_SKIP_LEVELS;
-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.PAY_CODEC;
-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.POS_CODEC;
-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.TERMS_CODEC;
-import static org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.VERSION_CURRENT;
-
-import java.io.IOException;
-import java.nio.ByteOrder;
-import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat.IntBlockTermState;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.BlockTermState;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
-import org.apache.lucene.codecs.PushPostingsWriterBase;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BitUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
-
-/**
- * Concrete class that writes docId(maybe frq,pos,offset,payloads) list with postings format.
- *
- * <p>Postings list for each term will be stored separately.
- *
- * @see Lucene84SkipWriter for details about skipping setting and postings layout.
- * @lucene.experimental
- */
-public final class Lucene84PostingsWriter extends PushPostingsWriterBase {
-
- IndexOutput docOut;
- IndexOutput posOut;
- IndexOutput payOut;
-
- static final IntBlockTermState emptyState = new IntBlockTermState();
- IntBlockTermState lastState;
-
- // Holds starting file pointers for current term:
- private long docStartFP;
- private long posStartFP;
- private long payStartFP;
-
- final long[] docDeltaBuffer;
- final long[] freqBuffer;
- private int docBufferUpto;
-
- final long[] posDeltaBuffer;
- final long[] payloadLengthBuffer;
- final long[] offsetStartDeltaBuffer;
- final long[] offsetLengthBuffer;
- private int posBufferUpto;
-
- private byte[] payloadBytes;
- private int payloadByteUpto;
-
- private int lastBlockDocID;
- private long lastBlockPosFP;
- private long lastBlockPayFP;
- private int lastBlockPosBufferUpto;
- private int lastBlockPayloadByteUpto;
-
- private int lastDocID;
- private int lastPosition;
- private int lastStartOffset;
- private int docCount;
-
- private final PForUtil pforUtil;
- private final ForDeltaUtil forDeltaUtil;
- private final Lucene84SkipWriter skipWriter;
-
- private boolean fieldHasNorms;
- private NumericDocValues norms;
- private final CompetitiveImpactAccumulator competitiveFreqNormAccumulator =
- new CompetitiveImpactAccumulator();
-
- /** Creates a postings writer */
- public Lucene84PostingsWriter(SegmentWriteState state) throws IOException {
-
- String docFileName =
- IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, Lucene84PostingsFormat.DOC_EXTENSION);
- docOut = EndiannessReverserUtil.createOutput(state.directory, docFileName, state.context);
- IndexOutput posOut = null;
- IndexOutput payOut = null;
- boolean success = false;
- try {
- CodecUtil.writeIndexHeader(
- docOut, DOC_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
- ByteOrder byteOrder = ByteOrder.nativeOrder();
- if (byteOrder == ByteOrder.BIG_ENDIAN) {
- docOut.writeByte((byte) 'B');
- } else if (byteOrder == ByteOrder.LITTLE_ENDIAN) {
- docOut.writeByte((byte) 'L');
- } else {
- throw new Error();
- }
- final ForUtil forUtil = new ForUtil();
- forDeltaUtil = new ForDeltaUtil(forUtil);
- pforUtil = new PForUtil(forUtil);
- if (state.fieldInfos.hasProx()) {
- posDeltaBuffer = new long[BLOCK_SIZE];
- String posFileName =
- IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, Lucene84PostingsFormat.POS_EXTENSION);
- posOut = EndiannessReverserUtil.createOutput(state.directory, posFileName, state.context);
- CodecUtil.writeIndexHeader(
- posOut, POS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
-
- if (state.fieldInfos.hasPayloads()) {
- payloadBytes = new byte[128];
- payloadLengthBuffer = new long[BLOCK_SIZE];
- } else {
- payloadBytes = null;
- payloadLengthBuffer = null;
- }
-
- if (state.fieldInfos.hasOffsets()) {
- offsetStartDeltaBuffer = new long[BLOCK_SIZE];
- offsetLengthBuffer = new long[BLOCK_SIZE];
- } else {
- offsetStartDeltaBuffer = null;
- offsetLengthBuffer = null;
- }
-
- if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {
- String payFileName =
- IndexFileNames.segmentFileName(
- state.segmentInfo.name,
- state.segmentSuffix,
- Lucene84PostingsFormat.PAY_EXTENSION);
- payOut = EndiannessReverserUtil.createOutput(state.directory, payFileName, state.context);
- CodecUtil.writeIndexHeader(
- payOut, PAY_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
- }
- } else {
- posDeltaBuffer = null;
- payloadLengthBuffer = null;
- offsetStartDeltaBuffer = null;
- offsetLengthBuffer = null;
- payloadBytes = null;
- }
- this.payOut = payOut;
- this.posOut = posOut;
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(docOut, posOut, payOut);
- }
- }
-
- docDeltaBuffer = new long[BLOCK_SIZE];
- freqBuffer = new long[BLOCK_SIZE];
-
- // TODO: should we try skipping every 2/4 blocks...?
- skipWriter =
- new Lucene84SkipWriter(
- MAX_SKIP_LEVELS, BLOCK_SIZE, state.segmentInfo.maxDoc(), docOut, posOut, payOut);
- }
-
- @Override
- public IntBlockTermState newTermState() {
- return new IntBlockTermState();
- }
-
- @Override
- public void init(IndexOutput termsOut, SegmentWriteState state) throws IOException {
- CodecUtil.writeIndexHeader(
- termsOut, TERMS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
- termsOut.writeVInt(BLOCK_SIZE);
- }
-
- @Override
- public void setField(FieldInfo fieldInfo) {
- super.setField(fieldInfo);
- skipWriter.setField(writePositions, writeOffsets, writePayloads);
- lastState = emptyState;
- fieldHasNorms = fieldInfo.hasNorms();
- }
-
- @Override
- public void startTerm(NumericDocValues norms) {
- docStartFP = docOut.getFilePointer();
- if (writePositions) {
- posStartFP = posOut.getFilePointer();
- if (writePayloads || writeOffsets) {
- payStartFP = payOut.getFilePointer();
- }
- }
- lastDocID = 0;
- lastBlockDocID = -1;
- skipWriter.resetSkip();
- this.norms = norms;
- competitiveFreqNormAccumulator.clear();
- }
-
- @Override
- public void startDoc(int docID, int termDocFreq) throws IOException {
- // Have collected a block of docs, and get a new doc.
- // Should write skip data as well as postings list for
- // current block.
- if (lastBlockDocID != -1 && docBufferUpto == 0) {
- skipWriter.bufferSkip(
- lastBlockDocID,
- competitiveFreqNormAccumulator,
- docCount,
- lastBlockPosFP,
- lastBlockPayFP,
- lastBlockPosBufferUpto,
- lastBlockPayloadByteUpto);
- competitiveFreqNormAccumulator.clear();
- }
-
- final int docDelta = docID - lastDocID;
-
- if (docID < 0 || (docCount > 0 && docDelta <= 0)) {
- throw new CorruptIndexException(
- "docs out of order (" + docID + " <= " + lastDocID + " )", docOut);
- }
-
- docDeltaBuffer[docBufferUpto] = docDelta;
- if (writeFreqs) {
- freqBuffer[docBufferUpto] = termDocFreq;
- }
-
- docBufferUpto++;
- docCount++;
-
- if (docBufferUpto == BLOCK_SIZE) {
- forDeltaUtil.encodeDeltas(docDeltaBuffer, docOut);
- if (writeFreqs) {
- pforUtil.encode(freqBuffer, docOut);
- }
- // NOTE: don't set docBufferUpto back to 0 here;
- // finishDoc will do so (because it needs to see that
- // the block was filled so it can save skip data)
- }
-
- lastDocID = docID;
- lastPosition = 0;
- lastStartOffset = 0;
-
- long norm;
- if (fieldHasNorms) {
- boolean found = norms.advanceExact(docID);
- if (found == false) {
- // This can happen if indexing hits a problem after adding a doc to the
- // postings but before buffering the norm. Such documents are written
- // deleted and will go away on the first merge.
- norm = 1L;
- } else {
- norm = norms.longValue();
- assert norm != 0 : docID;
- }
- } else {
- norm = 1L;
- }
-
- competitiveFreqNormAccumulator.add(writeFreqs ? termDocFreq : 1, norm);
- }
-
- @Override
- public void addPosition(int position, BytesRef payload, int startOffset, int endOffset)
- throws IOException {
- if (position > IndexWriter.MAX_POSITION) {
- throw new CorruptIndexException(
- "position="
- + position
- + " is too large (> IndexWriter.MAX_POSITION="
- + IndexWriter.MAX_POSITION
- + ")",
- docOut);
- }
- if (position < 0) {
- throw new CorruptIndexException("position=" + position + " is < 0", docOut);
- }
- posDeltaBuffer[posBufferUpto] = position - lastPosition;
- if (writePayloads) {
- if (payload == null || payload.length == 0) {
- // no payload
- payloadLengthBuffer[posBufferUpto] = 0;
- } else {
- payloadLengthBuffer[posBufferUpto] = payload.length;
- if (payloadByteUpto + payload.length > payloadBytes.length) {
- payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payload.length);
- }
- System.arraycopy(
- payload.bytes, payload.offset, payloadBytes, payloadByteUpto, payload.length);
- payloadByteUpto += payload.length;
- }
- }
-
- if (writeOffsets) {
- assert startOffset >= lastStartOffset;
- assert endOffset >= startOffset;
- offsetStartDeltaBuffer[posBufferUpto] = startOffset - lastStartOffset;
- offsetLengthBuffer[posBufferUpto] = endOffset - startOffset;
- lastStartOffset = startOffset;
- }
-
- posBufferUpto++;
- lastPosition = position;
- if (posBufferUpto == BLOCK_SIZE) {
- pforUtil.encode(posDeltaBuffer, posOut);
-
- if (writePayloads) {
- pforUtil.encode(payloadLengthBuffer, payOut);
- payOut.writeVInt(payloadByteUpto);
- payOut.writeBytes(payloadBytes, 0, payloadByteUpto);
- payloadByteUpto = 0;
- }
- if (writeOffsets) {
- pforUtil.encode(offsetStartDeltaBuffer, payOut);
- pforUtil.encode(offsetLengthBuffer, payOut);
- }
- posBufferUpto = 0;
- }
- }
-
- @Override
- public void finishDoc() throws IOException {
- // Since we don't know df for current term, we had to buffer
- // those skip data for each block, and when a new doc comes,
- // write them to skip file.
- if (docBufferUpto == BLOCK_SIZE) {
- lastBlockDocID = lastDocID;
- if (posOut != null) {
- if (payOut != null) {
- lastBlockPayFP = payOut.getFilePointer();
- }
- lastBlockPosFP = posOut.getFilePointer();
- lastBlockPosBufferUpto = posBufferUpto;
- lastBlockPayloadByteUpto = payloadByteUpto;
- }
- docBufferUpto = 0;
- }
- }
-
- /** Called when we are done adding docs to this term */
- @Override
- public void finishTerm(BlockTermState _state) throws IOException {
- IntBlockTermState state = (IntBlockTermState) _state;
- assert state.docFreq > 0;
-
- // TODO: wasteful we are counting this (counting # docs
- // for this term) in two places?
- assert state.docFreq == docCount : state.docFreq + " vs " + docCount;
-
- // docFreq == 1, don't write the single docid/freq to a separate file along with a pointer to
- // it.
- final int singletonDocID;
- if (state.docFreq == 1) {
- // pulse the singleton docid into the term dictionary, freq is implicitly totalTermFreq
- singletonDocID = (int) docDeltaBuffer[0];
- } else {
- singletonDocID = -1;
- // vInt encode the remaining doc deltas and freqs:
- for (int i = 0; i < docBufferUpto; i++) {
- final int docDelta = (int) docDeltaBuffer[i];
- final int freq = (int) freqBuffer[i];
- if (!writeFreqs) {
- docOut.writeVInt(docDelta);
- } else if (freq == 1) {
- docOut.writeVInt((docDelta << 1) | 1);
- } else {
- docOut.writeVInt(docDelta << 1);
- docOut.writeVInt(freq);
- }
- }
- }
-
- final long lastPosBlockOffset;
-
- if (writePositions) {
- // totalTermFreq is just total number of positions(or payloads, or offsets)
- // associated with current term.
- assert state.totalTermFreq != -1;
- if (state.totalTermFreq > BLOCK_SIZE) {
- // record file offset for last pos in last block
- lastPosBlockOffset = posOut.getFilePointer() - posStartFP;
- } else {
- lastPosBlockOffset = -1;
- }
- if (posBufferUpto > 0) {
- // TODO: should we send offsets/payloads to
- // .pay...? seems wasteful (have to store extra
- // vLong for low (< BLOCK_SIZE) DF terms = vast vast
- // majority)
-
- // vInt encode the remaining positions/payloads/offsets:
- int lastPayloadLength = -1; // force first payload length to be written
- int lastOffsetLength = -1; // force first offset length to be written
- int payloadBytesReadUpto = 0;
- for (int i = 0; i < posBufferUpto; i++) {
- final int posDelta = (int) posDeltaBuffer[i];
- if (writePayloads) {
- final int payloadLength = (int) payloadLengthBuffer[i];
- if (payloadLength != lastPayloadLength) {
- lastPayloadLength = payloadLength;
- posOut.writeVInt((posDelta << 1) | 1);
- posOut.writeVInt(payloadLength);
- } else {
- posOut.writeVInt(posDelta << 1);
- }
-
- if (payloadLength != 0) {
- posOut.writeBytes(payloadBytes, payloadBytesReadUpto, payloadLength);
- payloadBytesReadUpto += payloadLength;
- }
- } else {
- posOut.writeVInt(posDelta);
- }
-
- if (writeOffsets) {
- int delta = (int) offsetStartDeltaBuffer[i];
- int length = (int) offsetLengthBuffer[i];
- if (length == lastOffsetLength) {
- posOut.writeVInt(delta << 1);
- } else {
- posOut.writeVInt(delta << 1 | 1);
- posOut.writeVInt(length);
- lastOffsetLength = length;
- }
- }
- }
-
- if (writePayloads) {
- assert payloadBytesReadUpto == payloadByteUpto;
- payloadByteUpto = 0;
- }
- }
- } else {
- lastPosBlockOffset = -1;
- }
-
- long skipOffset;
- if (docCount > BLOCK_SIZE) {
- skipOffset = skipWriter.writeSkip(docOut) - docStartFP;
- } else {
- skipOffset = -1;
- }
-
- state.docStartFP = docStartFP;
- state.posStartFP = posStartFP;
- state.payStartFP = payStartFP;
- state.singletonDocID = singletonDocID;
- state.skipOffset = skipOffset;
- state.lastPosBlockOffset = lastPosBlockOffset;
- docBufferUpto = 0;
- posBufferUpto = 0;
- lastDocID = 0;
- docCount = 0;
- }
-
- @Override
- public void encodeTerm(
- DataOutput out, FieldInfo fieldInfo, BlockTermState _state, boolean absolute)
- throws IOException {
- IntBlockTermState state = (IntBlockTermState) _state;
- if (absolute) {
- lastState = emptyState;
- assert lastState.docStartFP == 0;
- }
-
- if (lastState.singletonDocID != -1
- && state.singletonDocID != -1
- && state.docStartFP == lastState.docStartFP) {
- // With runs of rare values such as ID fields, the increment of pointers in the docs file is
- // often 0.
- // Furthermore some ID schemes like auto-increment IDs or Flake IDs are monotonic, so we
- // encode the delta
- // between consecutive doc IDs to save space.
- final long delta = (long) state.singletonDocID - lastState.singletonDocID;
- out.writeVLong((BitUtil.zigZagEncode(delta) << 1) | 0x01);
- } else {
- out.writeVLong((state.docStartFP - lastState.docStartFP) << 1);
- if (state.singletonDocID != -1) {
- out.writeVInt(state.singletonDocID);
- }
- }
-
- if (writePositions) {
- out.writeVLong(state.posStartFP - lastState.posStartFP);
- if (writePayloads || writeOffsets) {
- out.writeVLong(state.payStartFP - lastState.payStartFP);
- }
- }
- if (writePositions) {
- if (state.lastPosBlockOffset != -1) {
- out.writeVLong(state.lastPosBlockOffset);
- }
- }
- if (state.skipOffset != -1) {
- out.writeVLong(state.skipOffset);
- }
- lastState = state;
- }
-
- @Override
- public void close() throws IOException {
- // TODO: add a finish() at least to PushBase? DV too...?
- boolean success = false;
- try {
- if (docOut != null) {
- CodecUtil.writeFooter(docOut);
- }
- if (posOut != null) {
- CodecUtil.writeFooter(posOut);
- }
- if (payOut != null) {
- CodecUtil.writeFooter(payOut);
- }
- success = true;
- } finally {
- if (success) {
- IOUtils.close(docOut, posOut, payOut);
- } else {
- IOUtils.closeWhileHandlingException(docOut, posOut, payOut);
- }
- docOut = posOut = payOut = null;
- }
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84RWCodec.java
deleted file mode 100644
index b129f5a2ac6..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84RWCodec.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene84;
-
-import org.apache.lucene.backward_codecs.lucene50.Lucene50RWCompoundFormat;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50RWStoredFieldsFormat;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50RWTermVectorsFormat;
-import org.apache.lucene.backward_codecs.lucene60.Lucene60RWPointsFormat;
-import org.apache.lucene.backward_codecs.lucene70.Lucene70RWSegmentInfoFormat;
-import org.apache.lucene.backward_codecs.lucene80.Lucene80RWNormsFormat;
-import org.apache.lucene.codecs.CompoundFormat;
-import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.PointsFormat;
-import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.SegmentInfoFormat;
-import org.apache.lucene.codecs.StoredFieldsFormat;
-import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
-
-/** RW impersonation of {@link Lucene84Codec}. */
-public class Lucene84RWCodec extends Lucene84Codec {
-
- private final PostingsFormat defaultPF = new Lucene84RWPostingsFormat();
- private final CompoundFormat compoundFormat = new Lucene50RWCompoundFormat();
- private final PostingsFormat postingsFormat =
- new PerFieldPostingsFormat() {
- @Override
- public PostingsFormat getPostingsFormatForField(String field) {
- return defaultPF;
- }
- };
-
- @Override
- public PointsFormat pointsFormat() {
- return new Lucene60RWPointsFormat();
- }
-
- @Override
- public SegmentInfoFormat segmentInfoFormat() {
- return new Lucene70RWSegmentInfoFormat();
- }
-
- @Override
- public StoredFieldsFormat storedFieldsFormat() {
- return new Lucene50RWStoredFieldsFormat();
- }
-
- @Override
- public NormsFormat normsFormat() {
- return new Lucene80RWNormsFormat();
- }
-
- @Override
- public PostingsFormat postingsFormat() {
- return postingsFormat;
- }
-
- @Override
- public final CompoundFormat compoundFormat() {
- return compoundFormat;
- }
-
- @Override
- public TermVectorsFormat termVectorsFormat() {
- return new Lucene50RWTermVectorsFormat();
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84RWPostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84RWPostingsFormat.java
deleted file mode 100644
index 0b88d6e7dcf..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84RWPostingsFormat.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene84;
-
-import java.io.IOException;
-import org.apache.lucene.backward_codecs.lucene40.blocktree.Lucene40BlockTreeTermsWriter;
-import org.apache.lucene.codecs.FieldsConsumer;
-import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.util.IOUtils;
-
-public class Lucene84RWPostingsFormat extends Lucene84PostingsFormat {
- @Override
- public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene84PostingsWriter(state);
- boolean success = false;
- try {
- FieldsConsumer ret =
- new Lucene40BlockTreeTermsWriter(
- state,
- postingsWriter,
- Lucene40BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,
- Lucene40BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
- success = true;
- return ret;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(postingsWriter);
- }
- }
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84SkipWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84SkipWriter.java
deleted file mode 100644
index 3d063f8c799..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84SkipWriter.java
+++ /dev/null
@@ -1,237 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene84;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
-import org.apache.lucene.codecs.Impact;
-import org.apache.lucene.codecs.MultiLevelSkipListWriter;
-import org.apache.lucene.store.ByteBuffersDataOutput;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.IndexOutput;
-
-/**
- * Write skip lists with multiple levels, and support skip within block ints.
- *
- * <p>Assume that docFreq = 28, skipInterval = blockSize = 12
- *
- * <pre>
- * | block#0 | | block#1 | |vInts|
- * d d d d d d d d d d d d d d d d d d d d d d d d d d d d (posting list)
- * ^ ^ (level 0 skip point)
- * </pre>
- *
- * <p>Note that skipWriter will ignore first document in block#0, since it is useless as a skip
- * point. Also, we'll never skip into the vInts block, only record skip data at the start its start
- * point(if it exist).
- *
- * <p>For each skip point, we will record: 1. docID in former position, i.e. for position 12, record
- * docID[11], etc. 2. its related file points(position, payload), 3. related numbers or
- * uptos(position, payload). 4. start offset.
- */
-final class Lucene84SkipWriter extends MultiLevelSkipListWriter {
- private int[] lastSkipDoc;
- private long[] lastSkipDocPointer;
- private long[] lastSkipPosPointer;
- private long[] lastSkipPayPointer;
- private int[] lastPayloadByteUpto;
-
- private final IndexOutput docOut;
- private final IndexOutput posOut;
- private final IndexOutput payOut;
-
- private int curDoc;
- private long curDocPointer;
- private long curPosPointer;
- private long curPayPointer;
- private int curPosBufferUpto;
- private int curPayloadByteUpto;
- private CompetitiveImpactAccumulator[] curCompetitiveFreqNorms;
- private boolean fieldHasPositions;
- private boolean fieldHasOffsets;
- private boolean fieldHasPayloads;
-
- public Lucene84SkipWriter(
- int maxSkipLevels,
- int blockSize,
- int docCount,
- IndexOutput docOut,
- IndexOutput posOut,
- IndexOutput payOut) {
- super(blockSize, 8, maxSkipLevels, docCount);
- this.docOut = docOut;
- this.posOut = posOut;
- this.payOut = payOut;
-
- lastSkipDoc = new int[maxSkipLevels];
- lastSkipDocPointer = new long[maxSkipLevels];
- if (posOut != null) {
- lastSkipPosPointer = new long[maxSkipLevels];
- if (payOut != null) {
- lastSkipPayPointer = new long[maxSkipLevels];
- }
- lastPayloadByteUpto = new int[maxSkipLevels];
- }
- curCompetitiveFreqNorms = new CompetitiveImpactAccumulator[maxSkipLevels];
- for (int i = 0; i < maxSkipLevels; ++i) {
- curCompetitiveFreqNorms[i] = new CompetitiveImpactAccumulator();
- }
- }
-
- public void setField(
- boolean fieldHasPositions, boolean fieldHasOffsets, boolean fieldHasPayloads) {
- this.fieldHasPositions = fieldHasPositions;
- this.fieldHasOffsets = fieldHasOffsets;
- this.fieldHasPayloads = fieldHasPayloads;
- }
-
- // tricky: we only skip data for blocks (terms with more than 128 docs), but re-init'ing the
- // skipper
- // is pretty slow for rare terms in large segments as we have to fill O(log #docs in segment) of
- // junk.
- // this is the vast majority of terms (worst case: ID field or similar). so in resetSkip() we
- // save
- // away the previous pointers, and lazy-init only if we need to buffer skip data for the term.
- private boolean initialized;
- long lastDocFP;
- long lastPosFP;
- long lastPayFP;
-
- @Override
- public void resetSkip() {
- lastDocFP = docOut.getFilePointer();
- if (fieldHasPositions) {
- lastPosFP = posOut.getFilePointer();
- if (fieldHasOffsets || fieldHasPayloads) {
- lastPayFP = payOut.getFilePointer();
- }
- }
- if (initialized) {
- for (CompetitiveImpactAccumulator acc : curCompetitiveFreqNorms) {
- acc.clear();
- }
- }
- initialized = false;
- }
-
- private void initSkip() {
- if (!initialized) {
- super.resetSkip();
- Arrays.fill(lastSkipDoc, 0);
- Arrays.fill(lastSkipDocPointer, lastDocFP);
- if (fieldHasPositions) {
- Arrays.fill(lastSkipPosPointer, lastPosFP);
- if (fieldHasPayloads) {
- Arrays.fill(lastPayloadByteUpto, 0);
- }
- if (fieldHasOffsets || fieldHasPayloads) {
- Arrays.fill(lastSkipPayPointer, lastPayFP);
- }
- }
- // sets of competitive freq,norm pairs should be empty at this point
- assert Arrays.stream(curCompetitiveFreqNorms)
- .map(CompetitiveImpactAccumulator::getCompetitiveFreqNormPairs)
- .mapToInt(Collection::size)
- .sum()
- == 0;
- initialized = true;
- }
- }
-
- /** Sets the values for the current skip data. */
- public void bufferSkip(
- int doc,
- CompetitiveImpactAccumulator competitiveFreqNorms,
- int numDocs,
- long posFP,
- long payFP,
- int posBufferUpto,
- int payloadByteUpto)
- throws IOException {
- initSkip();
- this.curDoc = doc;
- this.curDocPointer = docOut.getFilePointer();
- this.curPosPointer = posFP;
- this.curPayPointer = payFP;
- this.curPosBufferUpto = posBufferUpto;
- this.curPayloadByteUpto = payloadByteUpto;
- this.curCompetitiveFreqNorms[0].addAll(competitiveFreqNorms);
- bufferSkip(numDocs);
- }
-
- private final ByteBuffersDataOutput freqNormOut = ByteBuffersDataOutput.newResettableInstance();
-
- @Override
- protected void writeSkipData(int level, DataOutput skipBuffer) throws IOException {
-
- int delta = curDoc - lastSkipDoc[level];
-
- skipBuffer.writeVInt(delta);
- lastSkipDoc[level] = curDoc;
-
- skipBuffer.writeVLong(curDocPointer - lastSkipDocPointer[level]);
- lastSkipDocPointer[level] = curDocPointer;
-
- if (fieldHasPositions) {
-
- skipBuffer.writeVLong(curPosPointer - lastSkipPosPointer[level]);
- lastSkipPosPointer[level] = curPosPointer;
- skipBuffer.writeVInt(curPosBufferUpto);
-
- if (fieldHasPayloads) {
- skipBuffer.writeVInt(curPayloadByteUpto);
- }
-
- if (fieldHasOffsets || fieldHasPayloads) {
- skipBuffer.writeVLong(curPayPointer - lastSkipPayPointer[level]);
- lastSkipPayPointer[level] = curPayPointer;
- }
- }
-
- CompetitiveImpactAccumulator competitiveFreqNorms = curCompetitiveFreqNorms[level];
- assert competitiveFreqNorms.getCompetitiveFreqNormPairs().size() > 0;
- if (level + 1 < numberOfSkipLevels) {
- curCompetitiveFreqNorms[level + 1].addAll(competitiveFreqNorms);
- }
- writeImpacts(competitiveFreqNorms, freqNormOut);
- skipBuffer.writeVInt(Math.toIntExact(freqNormOut.size()));
- freqNormOut.copyTo(skipBuffer);
- freqNormOut.reset();
- competitiveFreqNorms.clear();
- }
-
- static void writeImpacts(CompetitiveImpactAccumulator acc, DataOutput out) throws IOException {
- Collection<Impact> impacts = acc.getCompetitiveFreqNormPairs();
- Impact previous = new Impact(0, 0);
- for (Impact impact : impacts) {
- assert impact.freq > previous.freq;
- assert Long.compareUnsigned(impact.norm, previous.norm) > 0;
- int freqDelta = impact.freq - previous.freq - 1;
- long normDelta = impact.norm - previous.norm - 1;
- if (normDelta == 0) {
- // most of time, norm only increases by 1, so we can fold everything in a single byte
- out.writeVInt(freqDelta << 1);
- } else {
- out.writeVInt((freqDelta << 1) | 1);
- out.writeZLong(normDelta);
- }
- previous = impact;
- }
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestForDeltaUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestForDeltaUtil.java
deleted file mode 100644
index 8217751151a..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestForDeltaUtil.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene84;
-
-import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
-import java.io.IOException;
-import java.util.Arrays;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.store.ByteBuffersDirectory;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-import org.apache.lucene.tests.util.TestUtil;
-import org.apache.lucene.util.packed.PackedInts;
-
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestForDeltaUtil extends LuceneTestCase {
-
- public void testEncodeDecode() throws IOException {
- final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);
- final int[] values = new int[iterations * ForUtil.BLOCK_SIZE];
-
- for (int i = 0; i < iterations; ++i) {
- final int bpv = TestUtil.nextInt(random(), 1, 31 - 7);
- for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
- values[i * ForUtil.BLOCK_SIZE + j] =
- RandomNumbers.randomIntBetween(random(), 1, (int) PackedInts.maxValue(bpv));
- }
- }
-
- final Directory d = new ByteBuffersDirectory();
- final long endPointer;
-
- {
- // encode
- IndexOutput out = EndiannessReverserUtil.createOutput(d, "test.bin", IOContext.DEFAULT);
- final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(new ForUtil());
-
- for (int i = 0; i < iterations; ++i) {
- long[] source = new long[ForUtil.BLOCK_SIZE];
- for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
- source[j] = values[i * ForUtil.BLOCK_SIZE + j];
- }
- forDeltaUtil.encodeDeltas(source, out);
- }
- endPointer = out.getFilePointer();
- out.close();
- }
-
- {
- // decode
- IndexInput in = EndiannessReverserUtil.openInput(d, "test.bin", IOContext.READONCE);
- final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(new ForUtil());
- for (int i = 0; i < iterations; ++i) {
- if (random().nextInt(5) == 0) {
- forDeltaUtil.skip(in);
- continue;
- }
- long base = 0;
- final long[] restored = new long[ForUtil.BLOCK_SIZE];
- forDeltaUtil.decodeAndPrefixSum(in, base, restored);
- final long[] expected = new long[ForUtil.BLOCK_SIZE];
- for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
- expected[j] = values[i * ForUtil.BLOCK_SIZE + j];
- if (j > 0) {
- expected[j] += expected[j - 1];
- } else {
- expected[j] += base;
- }
- }
- assertArrayEquals(Arrays.toString(restored), expected, restored);
- }
- assertEquals(endPointer, in.getFilePointer());
- in.close();
- }
-
- d.close();
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestForUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestForUtil.java
deleted file mode 100644
index 75aa44c9eb3..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestForUtil.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene84;
-
-import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
-import java.io.IOException;
-import java.util.Arrays;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.store.ByteBuffersDirectory;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-import org.apache.lucene.tests.util.TestUtil;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.packed.PackedInts;
-
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestForUtil extends LuceneTestCase {
-
- public void testEncodeDecode() throws IOException {
- final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);
- final int[] values = new int[iterations * ForUtil.BLOCK_SIZE];
-
- for (int i = 0; i < iterations; ++i) {
- final int bpv = TestUtil.nextInt(random(), 1, 31);
- for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
- values[i * ForUtil.BLOCK_SIZE + j] =
- RandomNumbers.randomIntBetween(random(), 0, (int) PackedInts.maxValue(bpv));
- }
- }
-
- final Directory d = new ByteBuffersDirectory();
- final long endPointer;
-
- {
- // encode
- IndexOutput out = EndiannessReverserUtil.createOutput(d, "test.bin", IOContext.DEFAULT);
- final ForUtil forUtil = new ForUtil();
-
- for (int i = 0; i < iterations; ++i) {
- long[] source = new long[ForUtil.BLOCK_SIZE];
- long or = 0;
- for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
- source[j] = values[i * ForUtil.BLOCK_SIZE + j];
- or |= source[j];
- }
- final int bpv = PackedInts.bitsRequired(or);
- out.writeByte((byte) bpv);
- forUtil.encode(source, bpv, out);
- }
- endPointer = out.getFilePointer();
- out.close();
- }
-
- {
- // decode
- IndexInput in = EndiannessReverserUtil.openInput(d, "test.bin", IOContext.READONCE);
- final ForUtil forUtil = new ForUtil();
- for (int i = 0; i < iterations; ++i) {
- final int bitsPerValue = in.readByte();
- final long currentFilePointer = in.getFilePointer();
- final long[] restored = new long[ForUtil.BLOCK_SIZE];
- forUtil.decode(bitsPerValue, in, restored);
- int[] ints = new int[ForUtil.BLOCK_SIZE];
- for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
- ints[j] = Math.toIntExact(restored[j]);
- }
- assertArrayEquals(
- Arrays.toString(ints),
- ArrayUtil.copyOfSubArray(values, i * ForUtil.BLOCK_SIZE, (i + 1) * ForUtil.BLOCK_SIZE),
- ints);
- assertEquals(forUtil.numBytes(bitsPerValue), in.getFilePointer() - currentFilePointer);
- }
- assertEquals(endPointer, in.getFilePointer());
- in.close();
- }
-
- d.close();
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestLucene84PostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestLucene84PostingsFormat.java
deleted file mode 100644
index 71b24c65b71..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestLucene84PostingsFormat.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene84;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import org.apache.lucene.backward_codecs.lucene40.blocktree.FieldReader;
-import org.apache.lucene.backward_codecs.lucene40.blocktree.Stats;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
-import org.apache.lucene.codecs.Impact;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.FreqAndNormBuffer;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.tests.analysis.MockAnalyzer;
-import org.apache.lucene.tests.index.BasePostingsFormatTestCase;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-import org.apache.lucene.tests.util.TestUtil;
-
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestLucene84PostingsFormat extends BasePostingsFormatTestCase {
- private final Codec codec = TestUtil.alwaysPostingsFormat(new Lucene84RWPostingsFormat());
-
- @Override
- protected Codec getCodec() {
- return codec;
- }
-
- /** Make sure the final sub-block(s) are not skipped. */
- public void testFinalBlock() throws Exception {
- Directory d = newDirectory();
- IndexWriter w = new IndexWriter(d, new IndexWriterConfig(new MockAnalyzer(random())));
- for (int i = 0; i < 25; i++) {
- Document doc = new Document();
- doc.add(newStringField("field", Character.toString((char) (97 + i)), Field.Store.NO));
- doc.add(newStringField("field", "z" + Character.toString((char) (97 + i)), Field.Store.NO));
- w.addDocument(doc);
- }
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- assertEquals(1, r.leaves().size());
- FieldReader field = (FieldReader) r.leaves().get(0).reader().terms("field");
- // We should see exactly two blocks: one root block (prefix empty string) and one block for z*
- // terms (prefix z):
- Stats stats = field.getStats();
- assertEquals(0, stats.floorBlockCount);
- assertEquals(2, stats.nonFloorBlockCount);
- r.close();
- w.close();
- d.close();
- }
-
- public void testImpactSerialization() throws IOException {
- // omit norms and omit freqs
- doTestImpactSerialization(Collections.singletonList(new Impact(1, 1L)));
-
- // omit freqs
- doTestImpactSerialization(Collections.singletonList(new Impact(1, 42L)));
- // omit freqs with very large norms
- doTestImpactSerialization(Collections.singletonList(new Impact(1, -100L)));
-
- // omit norms
- doTestImpactSerialization(Collections.singletonList(new Impact(30, 1L)));
- // omit norms with large freq
- doTestImpactSerialization(Collections.singletonList(new Impact(500, 1L)));
-
- // freqs and norms, basic
- doTestImpactSerialization(
- Arrays.asList(
- new Impact(1, 7L),
- new Impact(3, 9L),
- new Impact(7, 10L),
- new Impact(15, 11L),
- new Impact(20, 13L),
- new Impact(28, 14L)));
-
- // freqs and norms, high values
- doTestImpactSerialization(
- Arrays.asList(
- new Impact(2, 2L),
- new Impact(10, 10L),
- new Impact(12, 50L),
- new Impact(50, -100L),
- new Impact(1000, -80L),
- new Impact(1005, -3L)));
- }
-
- private void doTestImpactSerialization(List<Impact> impacts) throws IOException {
- CompetitiveImpactAccumulator acc = new CompetitiveImpactAccumulator();
- for (Impact impact : impacts) {
- acc.add(impact.freq, impact.norm);
- }
- try (Directory dir = newDirectory()) {
- try (IndexOutput out = EndiannessReverserUtil.createOutput(dir, "foo", IOContext.DEFAULT)) {
- Lucene84SkipWriter.writeImpacts(acc, out);
- }
- try (IndexInput in = EndiannessReverserUtil.openInput(dir, "foo", IOContext.DEFAULT)) {
- byte[] b = new byte[Math.toIntExact(in.length())];
- in.readBytes(b, 0, b.length);
- FreqAndNormBuffer impacts2 =
- Lucene84ScoreSkipReader.readImpacts(new ByteArrayDataInput(b), new FreqAndNormBuffer());
- List<Impact> impacts2AsList = new ArrayList<>();
- for (int i = 0; i < impacts2.size; ++i) {
- impacts2AsList.add(new Impact(impacts2.freqs[i], impacts2.norms[i]));
- }
- assertEquals(impacts, impacts2AsList);
- }
- }
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestPForUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestPForUtil.java
deleted file mode 100644
index fb6161830a3..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/TestPForUtil.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene84;
-
-import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
-import java.io.IOException;
-import java.util.Arrays;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.store.ByteBuffersDirectory;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-import org.apache.lucene.tests.util.TestUtil;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.packed.PackedInts;
-
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestPForUtil extends LuceneTestCase {
-
- public void testEncodeDecode() throws IOException {
- final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);
- final int[] values = new int[iterations * ForUtil.BLOCK_SIZE];
-
- for (int i = 0; i < iterations; ++i) {
- final int bpv = TestUtil.nextInt(random(), 0, 31);
- for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
- values[i * ForUtil.BLOCK_SIZE + j] =
- RandomNumbers.randomIntBetween(random(), 0, (int) PackedInts.maxValue(bpv));
- if (random().nextInt(100) == 0) {
- final int exceptionBpv;
- if (random().nextInt(10) == 0) {
- exceptionBpv = Math.min(bpv + TestUtil.nextInt(random(), 9, 16), 31);
- } else {
- exceptionBpv = Math.min(bpv + TestUtil.nextInt(random(), 1, 8), 31);
- }
- values[i * ForUtil.BLOCK_SIZE + j] |= random().nextInt(1 << (exceptionBpv - bpv)) << bpv;
- }
- }
- }
-
- final Directory d = new ByteBuffersDirectory();
- final long endPointer;
-
- {
- // encode
- IndexOutput out = EndiannessReverserUtil.createOutput(d, "test.bin", IOContext.DEFAULT);
- final PForUtil pforUtil = new PForUtil(new ForUtil());
-
- for (int i = 0; i < iterations; ++i) {
- long[] source = new long[ForUtil.BLOCK_SIZE];
- for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
- source[j] = values[i * ForUtil.BLOCK_SIZE + j];
- }
- pforUtil.encode(source, out);
- }
- endPointer = out.getFilePointer();
- out.close();
- }
-
- {
- // decode
- IndexInput in = EndiannessReverserUtil.openInput(d, "test.bin", IOContext.READONCE);
- final PForUtil pforUtil = new PForUtil(new ForUtil());
- for (int i = 0; i < iterations; ++i) {
- if (random().nextInt(5) == 0) {
- pforUtil.skip(in);
- continue;
- }
- final long[] restored = new long[ForUtil.BLOCK_SIZE];
- pforUtil.decode(in, restored);
- int[] ints = new int[ForUtil.BLOCK_SIZE];
- for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
- ints[j] = Math.toIntExact(restored[j]);
- }
- assertArrayEquals(
- Arrays.toString(ints),
- ArrayUtil.copyOfSubArray(values, i * ForUtil.BLOCK_SIZE, (i + 1) * ForUtil.BLOCK_SIZE),
- ints);
- }
- assertEquals(endPointer, in.getFilePointer());
- in.close();
- }
-
- d.close();
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86PointsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86PointsWriter.java
deleted file mode 100644
index d0bcc935811..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86PointsWriter.java
+++ /dev/null
@@ -1,304 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene86;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.MutablePointTree;
-import org.apache.lucene.codecs.PointsReader;
-import org.apache.lucene.codecs.PointsWriter;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.MergeState;
-import org.apache.lucene.index.PointValues;
-import org.apache.lucene.index.PointValues.IntersectVisitor;
-import org.apache.lucene.index.PointValues.Relation;
-import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.IORunnable;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.bkd.BKDConfig;
-import org.apache.lucene.util.bkd.BKDWriter;
-
-/** Writes dimensional values */
-public class Lucene86PointsWriter extends PointsWriter {
-
- /** Outputs used to write the BKD tree data files. */
- protected final IndexOutput metaOut, indexOut, dataOut;
-
- final SegmentWriteState writeState;
- final int maxPointsInLeafNode;
- final double maxMBSortInHeap;
- private boolean finished;
-
- /** Full constructor */
- public Lucene86PointsWriter(
- SegmentWriteState writeState, int maxPointsInLeafNode, double maxMBSortInHeap)
- throws IOException {
- assert writeState.fieldInfos.hasPointValues();
- this.writeState = writeState;
- this.maxPointsInLeafNode = maxPointsInLeafNode;
- this.maxMBSortInHeap = maxMBSortInHeap;
- String dataFileName =
- IndexFileNames.segmentFileName(
- writeState.segmentInfo.name,
- writeState.segmentSuffix,
- Lucene86PointsFormat.DATA_EXTENSION);
- dataOut =
- EndiannessReverserUtil.createOutput(writeState.directory, dataFileName, writeState.context);
- boolean success = false;
- try {
- CodecUtil.writeIndexHeader(
- dataOut,
- Lucene86PointsFormat.DATA_CODEC_NAME,
- Lucene86PointsFormat.VERSION_CURRENT,
- writeState.segmentInfo.getId(),
- writeState.segmentSuffix);
-
- String metaFileName =
- IndexFileNames.segmentFileName(
- writeState.segmentInfo.name,
- writeState.segmentSuffix,
- Lucene86PointsFormat.META_EXTENSION);
- metaOut =
- EndiannessReverserUtil.createOutput(
- writeState.directory, metaFileName, writeState.context);
- CodecUtil.writeIndexHeader(
- metaOut,
- Lucene86PointsFormat.META_CODEC_NAME,
- Lucene86PointsFormat.VERSION_CURRENT,
- writeState.segmentInfo.getId(),
- writeState.segmentSuffix);
-
- String indexFileName =
- IndexFileNames.segmentFileName(
- writeState.segmentInfo.name,
- writeState.segmentSuffix,
- Lucene86PointsFormat.INDEX_EXTENSION);
- indexOut =
- EndiannessReverserUtil.createOutput(
- writeState.directory, indexFileName, writeState.context);
- CodecUtil.writeIndexHeader(
- indexOut,
- Lucene86PointsFormat.INDEX_CODEC_NAME,
- Lucene86PointsFormat.VERSION_CURRENT,
- writeState.segmentInfo.getId(),
- writeState.segmentSuffix);
-
- success = true;
- } finally {
- if (success == false) {
- IOUtils.closeWhileHandlingException(this);
- }
- }
- }
-
- /**
- * Uses the defaults values for {@code maxPointsInLeafNode} (512) and {@code maxMBSortInHeap}
- * (16.0)
- */
- public Lucene86PointsWriter(SegmentWriteState writeState) throws IOException {
- this(
- writeState,
- BKDConfig.DEFAULT_MAX_POINTS_IN_LEAF_NODE,
- BKDWriter.DEFAULT_MAX_MB_SORT_IN_HEAP);
- }
-
- @Override
- public void writeField(FieldInfo fieldInfo, PointsReader reader) throws IOException {
-
- PointValues.PointTree values = reader.getValues(fieldInfo.name).getPointTree();
-
- BKDConfig config =
- new BKDConfig(
- fieldInfo.getPointDimensionCount(),
- fieldInfo.getPointIndexDimensionCount(),
- fieldInfo.getPointNumBytes(),
- maxPointsInLeafNode);
-
- try (BKDWriter writer =
- new BKDWriter(
- writeState.segmentInfo.maxDoc(),
- writeState.directory,
- writeState.segmentInfo.name,
- config,
- maxMBSortInHeap,
- values.size())) {
-
- if (values instanceof MutablePointTree) {
- IORunnable finalizer =
- writer.writeField(
- metaOut, indexOut, dataOut, fieldInfo.name, (MutablePointTree) values);
- if (finalizer != null) {
- metaOut.writeInt(fieldInfo.number);
- finalizer.run();
- }
- return;
- }
-
- values.visitDocValues(
- new IntersectVisitor() {
- @Override
- public void visit(int docID) {
- throw new IllegalStateException();
- }
-
- @Override
- public void visit(int docID, byte[] packedValue) throws IOException {
- writer.add(packedValue, docID);
- }
-
- @Override
- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- return Relation.CELL_CROSSES_QUERY;
- }
- });
-
- // We could have 0 points on merge since all docs with dimensional fields may be deleted:
- IORunnable finalizer = writer.finish(metaOut, indexOut, dataOut);
- if (finalizer != null) {
- metaOut.writeInt(fieldInfo.number);
- finalizer.run();
- }
- }
- }
-
- @Override
- public void merge(MergeState mergeState) throws IOException {
- /*
- * If indexSort is activated and some of the leaves are not sorted the next test will catch that
- * and the non-optimized merge will run. If the readers are all sorted then it's safe to perform
- * a bulk merge of the points.
- */
- for (PointsReader reader : mergeState.pointsReaders) {
- if (reader instanceof Lucene86PointsReader == false) {
- // We can only bulk merge when all to-be-merged segments use our format:
- super.merge(mergeState);
- return;
- }
- }
- for (PointsReader reader : mergeState.pointsReaders) {
- if (reader != null) {
- reader.checkIntegrity();
- }
- }
-
- for (FieldInfo fieldInfo : mergeState.mergeFieldInfos) {
- if (fieldInfo.getPointDimensionCount() != 0) {
- if (fieldInfo.getPointDimensionCount() == 1) {
-
- // Worst case total maximum size (if none of the points are deleted):
- long totMaxSize = 0;
- for (int i = 0; i < mergeState.pointsReaders.length; i++) {
- PointsReader reader = mergeState.pointsReaders[i];
- if (reader != null) {
- FieldInfos readerFieldInfos = mergeState.fieldInfos[i];
- FieldInfo readerFieldInfo = readerFieldInfos.fieldInfo(fieldInfo.name);
- if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {
- PointValues values = reader.getValues(fieldInfo.name);
- if (values != null) {
- totMaxSize += values.size();
- }
- }
- }
- }
-
- BKDConfig config =
- new BKDConfig(
- fieldInfo.getPointDimensionCount(),
- fieldInfo.getPointIndexDimensionCount(),
- fieldInfo.getPointNumBytes(),
- maxPointsInLeafNode);
-
- // System.out.println("MERGE: field=" + fieldInfo.name);
- // Optimize the 1D case to use BKDWriter.merge, which does a single merge sort of the
- // already sorted incoming segments, instead of trying to sort all points again as if
- // we were simply reindexing them:
- try (BKDWriter writer =
- new BKDWriter(
- writeState.segmentInfo.maxDoc(),
- writeState.directory,
- writeState.segmentInfo.name,
- config,
- maxMBSortInHeap,
- totMaxSize)) {
- List<PointValues> pointValues = new ArrayList<>();
- List<MergeState.DocMap> docMaps = new ArrayList<>();
- for (int i = 0; i < mergeState.pointsReaders.length; i++) {
- PointsReader reader = mergeState.pointsReaders[i];
-
- if (reader != null) {
-
- // we confirmed this up above
- assert reader instanceof Lucene86PointsReader;
- Lucene86PointsReader reader86 = (Lucene86PointsReader) reader;
-
- // NOTE: we cannot just use the merged fieldInfo.number (instead of resolving to
- // this
- // reader's FieldInfo as we do below) because field numbers can easily be different
- // when addIndexes(Directory...) copies over segments from another index:
-
- FieldInfos readerFieldInfos = mergeState.fieldInfos[i];
- FieldInfo readerFieldInfo = readerFieldInfos.fieldInfo(fieldInfo.name);
- if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {
- PointValues aPointValues = reader86.getValues(readerFieldInfo.name);
- if (aPointValues != null) {
- pointValues.add(aPointValues);
- docMaps.add(mergeState.docMaps[i]);
- }
- }
- }
- }
-
- IORunnable finalizer = writer.merge(metaOut, indexOut, dataOut, docMaps, pointValues);
- if (finalizer != null) {
- metaOut.writeInt(fieldInfo.number);
- finalizer.run();
- }
- }
- } else {
- mergeOneField(mergeState, fieldInfo);
- }
- }
- }
-
- finish();
- }
-
- @Override
- public void finish() throws IOException {
- if (finished) {
- throw new IllegalStateException("already finished");
- }
- finished = true;
- metaOut.writeInt(-1);
- CodecUtil.writeFooter(indexOut);
- CodecUtil.writeFooter(dataOut);
- metaOut.writeLong(indexOut.getFilePointer());
- metaOut.writeLong(dataOut.getFilePointer());
- CodecUtil.writeFooter(metaOut);
- }
-
- @Override
- public void close() throws IOException {
- IOUtils.close(metaOut, indexOut, dataOut);
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86RWCodec.java
deleted file mode 100644
index 60cf10b2dd8..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86RWCodec.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene86;
-
-import org.apache.lucene.backward_codecs.lucene50.Lucene50RWCompoundFormat;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50RWStoredFieldsFormat;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50RWTermVectorsFormat;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat;
-import org.apache.lucene.backward_codecs.lucene80.Lucene80RWNormsFormat;
-import org.apache.lucene.backward_codecs.lucene84.Lucene84RWPostingsFormat;
-import org.apache.lucene.codecs.CompoundFormat;
-import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.PointsFormat;
-import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.SegmentInfoFormat;
-import org.apache.lucene.codecs.StoredFieldsFormat;
-import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
-
-/** RW impersonation of {@link Lucene86Codec}. */
-public class Lucene86RWCodec extends Lucene86Codec {
-
- private final StoredFieldsFormat storedFieldsFormat;
- private final PostingsFormat defaultPF = new Lucene84RWPostingsFormat();
- private final CompoundFormat compoundFormat = new Lucene50RWCompoundFormat();
- private final PostingsFormat postingsFormat =
- new PerFieldPostingsFormat() {
- @Override
- public PostingsFormat getPostingsFormatForField(String field) {
- return defaultPF;
- }
- };
-
- /** No arguments constructor. */
- public Lucene86RWCodec() {
- storedFieldsFormat = new Lucene50RWStoredFieldsFormat();
- }
-
- /** Constructor that takes a mode. */
- public Lucene86RWCodec(Lucene50StoredFieldsFormat.Mode mode) {
- storedFieldsFormat = new Lucene50RWStoredFieldsFormat(mode);
- }
-
- @Override
- public SegmentInfoFormat segmentInfoFormat() {
- return new Lucene86RWSegmentInfoFormat();
- }
-
- @Override
- public StoredFieldsFormat storedFieldsFormat() {
- return storedFieldsFormat;
- }
-
- @Override
- public NormsFormat normsFormat() {
- return new Lucene80RWNormsFormat();
- }
-
- @Override
- public PostingsFormat postingsFormat() {
- return postingsFormat;
- }
-
- @Override
- public final CompoundFormat compoundFormat() {
- return compoundFormat;
- }
-
- @Override
- public TermVectorsFormat termVectorsFormat() {
- return new Lucene50RWTermVectorsFormat();
- }
-
- @Override
- public final PointsFormat pointsFormat() {
- return new Lucene86RWPointsFormat();
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86RWPointsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86RWPointsFormat.java
deleted file mode 100644
index 9fd239d090d..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86RWPointsFormat.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene86;
-
-import java.io.IOException;
-import org.apache.lucene.codecs.PointsWriter;
-import org.apache.lucene.index.SegmentWriteState;
-
-/** Writable version of Lucene86PointsFormat for testing */
-public final class Lucene86RWPointsFormat extends Lucene86PointsFormat {
-
- /** Sole constructor */
- public Lucene86RWPointsFormat() {}
-
- @Override
- public PointsWriter fieldsWriter(SegmentWriteState state) throws IOException {
- return new Lucene86PointsWriter(state);
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86RWSegmentInfoFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86RWSegmentInfoFormat.java
deleted file mode 100644
index ca90cec4c6a..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86RWSegmentInfoFormat.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.lucene.backward_codecs.lucene86;
-
-import java.io.IOException;
-import java.util.Set;
-import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.IndexSorter;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.SortFieldProvider;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.Version;
-
-/** Writable version of Lucene86SegmentInfoFormat for testing */
-public class Lucene86RWSegmentInfoFormat extends Lucene86SegmentInfoFormat {
-
- /** Sole constructor. */
- public Lucene86RWSegmentInfoFormat() {}
-
- @Override
- public void write(Directory dir, SegmentInfo si, IOContext ioContext) throws IOException {
- final String fileName = IndexFileNames.segmentFileName(si.name, "", SI_EXTENSION);
-
- try (IndexOutput output = EndiannessReverserUtil.createOutput(dir, fileName, ioContext)) {
- // Only add the file once we've successfully created it, else IFD assert can trip:
- si.addFile(fileName);
- CodecUtil.writeIndexHeader(output, CODEC_NAME, VERSION_CURRENT, si.getId(), "");
-
- writeSegmentInfo(output, si);
-
- CodecUtil.writeFooter(output);
- }
- }
-
- private void writeSegmentInfo(DataOutput output, SegmentInfo si) throws IOException {
- Version version = si.getVersion();
- if (version.major < 7) {
- throw new IllegalArgumentException(
- "invalid major version: should be >= 7 but got: " + version.major + " segment=" + si);
- }
- // Write the Lucene version that created this segment, since 3.1
- output.writeInt(version.major);
- output.writeInt(version.minor);
- output.writeInt(version.bugfix);
-
- // Write the min Lucene version that contributed docs to the segment, since 7.0
- if (si.getMinVersion() != null) {
- output.writeByte((byte) 1);
- Version minVersion = si.getMinVersion();
- output.writeInt(minVersion.major);
- output.writeInt(minVersion.minor);
- output.writeInt(minVersion.bugfix);
- } else {
- output.writeByte((byte) 0);
- }
-
- assert version.prerelease == 0;
- output.writeInt(si.maxDoc());
-
- output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO));
- output.writeMapOfStrings(si.getDiagnostics());
- Set<String> files = si.files();
- for (String file : files) {
- if (!IndexFileNames.parseSegmentName(file).equals(si.name)) {
- throw new IllegalArgumentException(
- "invalid files: expected segment=" + si.name + ", got=" + files);
- }
- }
- output.writeSetOfStrings(files);
- output.writeMapOfStrings(si.getAttributes());
-
- Sort indexSort = si.getIndexSort();
- int numSortFields = indexSort == null ? 0 : indexSort.getSort().length;
- output.writeVInt(numSortFields);
- for (int i = 0; i < numSortFields; ++i) {
- SortField sortField = indexSort.getSort()[i];
- IndexSorter sorter = sortField.getIndexSorter();
- if (sorter == null) {
- throw new IllegalArgumentException("cannot serialize SortField " + sortField);
- }
- output.writeString(sorter.getProviderName());
- SortFieldProvider.write(sortField, output);
- }
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/TestLucene86PointsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/TestLucene86PointsFormat.java
deleted file mode 100644
index b59ec079d4a..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/TestLucene86PointsFormat.java
+++ /dev/null
@@ -1,361 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene86;
-
-import java.io.IOException;
-import java.util.Arrays;
-import org.apache.lucene.backward_codecs.lucene87.Lucene87RWCodec;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.FilterCodec;
-import org.apache.lucene.codecs.PointsFormat;
-import org.apache.lucene.codecs.PointsReader;
-import org.apache.lucene.codecs.PointsWriter;
-import org.apache.lucene.document.BinaryPoint;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.PointValues;
-import org.apache.lucene.index.PointValues.IntersectVisitor;
-import org.apache.lucene.index.PointValues.Relation;
-import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.tests.index.BasePointsFormatTestCase;
-import org.apache.lucene.tests.index.MockRandomMergePolicy;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-import org.apache.lucene.tests.util.TestUtil;
-import org.apache.lucene.util.bkd.BKDConfig;
-
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestLucene86PointsFormat extends BasePointsFormatTestCase {
-
- private final Codec codec;
- private final int maxPointsInLeafNode;
-
- public TestLucene86PointsFormat() {
- // standard issue
- Codec defaultCodec = new Lucene87RWCodec();
- if (random().nextBoolean()) {
- // randomize parameters
- maxPointsInLeafNode = TestUtil.nextInt(random(), 50, 500);
- double maxMBSortInHeap = 3.0 + (3 * random().nextDouble());
- if (VERBOSE) {
- System.out.println(
- "TEST: using Lucene60PointsFormat with maxPointsInLeafNode="
- + maxPointsInLeafNode
- + " and maxMBSortInHeap="
- + maxMBSortInHeap);
- }
-
- // sneaky impersonation!
- codec =
- new FilterCodec(defaultCodec.getName(), defaultCodec) {
- @Override
- public PointsFormat pointsFormat() {
- return new PointsFormat() {
- @Override
- public PointsWriter fieldsWriter(SegmentWriteState writeState) throws IOException {
- return new Lucene86PointsWriter(writeState, maxPointsInLeafNode, maxMBSortInHeap);
- }
-
- @Override
- public PointsReader fieldsReader(SegmentReadState readState) throws IOException {
- return new Lucene86PointsReader(readState);
- }
- };
- }
- };
- } else {
- // standard issue
- codec = defaultCodec;
- maxPointsInLeafNode = BKDConfig.DEFAULT_MAX_POINTS_IN_LEAF_NODE;
- }
- }
-
- @Override
- protected Codec getCodec() {
- return codec;
- }
-
- @Override
- public void testMergeStability() throws Exception {
- assumeFalse(
- "TODO: mess with the parameters and test gets angry!", codec instanceof FilterCodec);
- super.testMergeStability();
- }
-
- // TODO: clean up the math/estimation here rather than suppress so many warnings
- @SuppressWarnings({"NarrowCalculation", "LongDoubleConversion"})
- public void testEstimatePointCount() throws IOException {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig();
- // Avoid mockRandomMP since it may cause non-optimal merges that make the
- // number of points per leaf hard to predict
- while (iwc.getMergePolicy() instanceof MockRandomMergePolicy) {
- iwc.setMergePolicy(newMergePolicy());
- }
- IndexWriter w = new IndexWriter(dir, iwc);
- byte[] pointValue = new byte[3];
- byte[] uniquePointValue = new byte[3];
- random().nextBytes(uniquePointValue);
- final int numDocs =
- TEST_NIGHTLY ? atLeast(10000) : atLeast(500); // at night, make sure we have several leaves
- final boolean multiValues = random().nextBoolean();
- int totalValues = 0;
- for (int i = 0; i < numDocs; ++i) {
- Document doc = new Document();
- if (i == numDocs / 2) {
- totalValues++;
- doc.add(new BinaryPoint("f", uniquePointValue));
- } else {
- final int numValues = (multiValues) ? TestUtil.nextInt(random(), 2, 100) : 1;
- for (int j = 0; j < numValues; j++) {
- do {
- random().nextBytes(pointValue);
- } while (Arrays.equals(pointValue, uniquePointValue));
- doc.add(new BinaryPoint("f", pointValue));
- totalValues++;
- }
- }
- w.addDocument(doc);
- }
- w.forceMerge(1);
- final IndexReader r = DirectoryReader.open(w);
- w.close();
- final LeafReader lr = getOnlyLeafReader(r);
- PointValues points = lr.getPointValues("f");
-
- IntersectVisitor allPointsVisitor =
- new IntersectVisitor() {
- @Override
- public void visit(int docID, byte[] packedValue) throws IOException {}
-
- @Override
- public void visit(int docID) throws IOException {}
-
- @Override
- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- return Relation.CELL_INSIDE_QUERY;
- }
- };
-
- assertEquals(totalValues, points.estimatePointCount(allPointsVisitor));
- assertEquals(numDocs, points.estimateDocCount(allPointsVisitor));
-
- IntersectVisitor noPointsVisitor =
- new IntersectVisitor() {
- @Override
- public void visit(int docID, byte[] packedValue) throws IOException {}
-
- @Override
- public void visit(int docID) throws IOException {}
-
- @Override
- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- return Relation.CELL_OUTSIDE_QUERY;
- }
- };
-
- // Return 0 if no points match
- assertEquals(0, points.estimatePointCount(noPointsVisitor));
- assertEquals(0, points.estimateDocCount(noPointsVisitor));
-
- IntersectVisitor onePointMatchVisitor =
- new IntersectVisitor() {
- @Override
- public void visit(int docID, byte[] packedValue) throws IOException {}
-
- @Override
- public void visit(int docID) throws IOException {}
-
- @Override
- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- if (Arrays.compareUnsigned(uniquePointValue, 0, 3, maxPackedValue, 0, 3) > 0
- || Arrays.compareUnsigned(uniquePointValue, 0, 3, minPackedValue, 0, 3) < 0) {
- return Relation.CELL_OUTSIDE_QUERY;
- }
- return Relation.CELL_CROSSES_QUERY;
- }
- };
-
- // If only one point matches, then the point count is (maxPointsInLeafNode + 1) / 2
- // in general, or maybe 2x that if the point is a split value
- final long pointCount = points.estimatePointCount(onePointMatchVisitor);
- final long lastNodePointCount = totalValues % maxPointsInLeafNode;
- assertTrue(
- "" + pointCount,
- pointCount == (maxPointsInLeafNode + 1) / 2 // common case
- || pointCount == (lastNodePointCount + 1) / 2 // not fully populated leaf
- || pointCount == 2 * ((maxPointsInLeafNode + 1) / 2) // if the point is a split value
- || pointCount
- == ((maxPointsInLeafNode + 1) / 2)
- + ((lastNodePointCount + 1)
- / 2)); // if the point is a split value and one leaf is not fully populated
-
- final long docCount = points.estimateDocCount(onePointMatchVisitor);
-
- if (multiValues) {
- assertEquals(
- docCount,
- (long)
- (docCount
- * (1d
- - Math.pow(
- (numDocs - pointCount) / points.size(), points.size() / docCount))));
- } else {
- assertEquals(Math.min(pointCount, numDocs), docCount);
- }
- r.close();
- dir.close();
- }
-
- // The tree is always balanced in the N dims case, and leaves are
- // not all full so things are a bit different
- // TODO: clean up the math/estimation here rather than suppress so many warnings
- @SuppressWarnings({"NarrowCalculation", "LongDoubleConversion"})
- public void testEstimatePointCount2Dims() throws IOException {
- Directory dir = newDirectory();
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
- byte[][] pointValue = new byte[2][];
- pointValue[0] = new byte[3];
- pointValue[1] = new byte[3];
- byte[][] uniquePointValue = new byte[2][];
- uniquePointValue[0] = new byte[3];
- uniquePointValue[1] = new byte[3];
- random().nextBytes(uniquePointValue[0]);
- random().nextBytes(uniquePointValue[1]);
- final int numDocs =
- TEST_NIGHTLY
- ? atLeast(10000)
- : atLeast(1000); // in nightly, make sure we have several leaves
- final boolean multiValues = random().nextBoolean();
- int totalValues = 0;
- for (int i = 0; i < numDocs; ++i) {
- Document doc = new Document();
- if (i == numDocs / 2) {
- doc.add(new BinaryPoint("f", uniquePointValue));
- totalValues++;
- } else {
- final int numValues = (multiValues) ? TestUtil.nextInt(random(), 2, 100) : 1;
- for (int j = 0; j < numValues; j++) {
- do {
- random().nextBytes(pointValue[0]);
- random().nextBytes(pointValue[1]);
- } while (Arrays.equals(pointValue[0], uniquePointValue[0])
- || Arrays.equals(pointValue[1], uniquePointValue[1]));
- doc.add(new BinaryPoint("f", pointValue));
- totalValues++;
- }
- }
- w.addDocument(doc);
- }
- w.forceMerge(1);
- final IndexReader r = DirectoryReader.open(w);
- w.close();
- final LeafReader lr = getOnlyLeafReader(r);
- PointValues points = lr.getPointValues("f");
-
- IntersectVisitor allPointsVisitor =
- new IntersectVisitor() {
- @Override
- public void visit(int docID, byte[] packedValue) throws IOException {}
-
- @Override
- public void visit(int docID) throws IOException {}
-
- @Override
- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- return Relation.CELL_INSIDE_QUERY;
- }
- };
-
- assertEquals(totalValues, points.estimatePointCount(allPointsVisitor));
- assertEquals(numDocs, points.estimateDocCount(allPointsVisitor));
-
- IntersectVisitor noPointsVisitor =
- new IntersectVisitor() {
- @Override
- public void visit(int docID, byte[] packedValue) throws IOException {}
-
- @Override
- public void visit(int docID) throws IOException {}
-
- @Override
- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- return Relation.CELL_OUTSIDE_QUERY;
- }
- };
-
- // Return 0 if no points match
- assertEquals(0, points.estimatePointCount(noPointsVisitor));
- assertEquals(0, points.estimateDocCount(noPointsVisitor));
-
- IntersectVisitor onePointMatchVisitor =
- new IntersectVisitor() {
- @Override
- public void visit(int docID, byte[] packedValue) throws IOException {}
-
- @Override
- public void visit(int docID) throws IOException {}
-
- @Override
- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- for (int dim = 0; dim < 2; ++dim) {
- if (Arrays.compareUnsigned(
- uniquePointValue[dim], 0, 3, maxPackedValue, dim * 3, dim * 3 + 3)
- > 0
- || Arrays.compareUnsigned(
- uniquePointValue[dim], 0, 3, minPackedValue, dim * 3, dim * 3 + 3)
- < 0) {
- return Relation.CELL_OUTSIDE_QUERY;
- }
- }
- return Relation.CELL_CROSSES_QUERY;
- }
- };
-
- final long pointCount = points.estimatePointCount(onePointMatchVisitor);
- final long lastNodePointCount = totalValues % maxPointsInLeafNode;
- assertTrue(
- "" + pointCount,
- pointCount == (maxPointsInLeafNode + 1) / 2 // common case
- || pointCount == (lastNodePointCount + 1) / 2 // not fully populated leaf
- || pointCount == 2 * ((maxPointsInLeafNode + 1) / 2) // if the point is a split value
- || pointCount == ((maxPointsInLeafNode + 1) / 2) + ((lastNodePointCount + 1) / 2)
- // in extreme cases, a point can be shared by 4 leaves
- || pointCount == 4 * ((maxPointsInLeafNode + 1) / 2)
- || pointCount == 3 * ((maxPointsInLeafNode + 1) / 2) + ((lastNodePointCount + 1) / 2));
-
- final long docCount = points.estimateDocCount(onePointMatchVisitor);
- if (multiValues) {
- assertEquals(
- docCount,
- (long)
- (docCount
- * (1d
- - Math.pow(
- (numDocs - pointCount) / points.size(), points.size() / docCount))));
- } else {
- assertEquals(Math.min(pointCount, numDocs), docCount);
- }
- r.close();
- dir.close();
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/TestLucene86SegmentInfoFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/TestLucene86SegmentInfoFormat.java
deleted file mode 100644
index 7388dcf8210..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/TestLucene86SegmentInfoFormat.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.lucene.backward_codecs.lucene86;
-
-import org.apache.lucene.backward_codecs.lucene87.Lucene87RWCodec;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.tests.index.BaseSegmentInfoFormatTestCase;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-import org.apache.lucene.util.Version;
-
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestLucene86SegmentInfoFormat extends BaseSegmentInfoFormatTestCase {
-
- @Override
- protected Version[] getVersions() {
- return new Version[] {Version.fromBits(8, 8, 1)};
- }
-
- @Override
- protected Codec getCodec() {
- return new Lucene87RWCodec();
- }
-
- @Override
- protected boolean supportsHasBlocks() {
- return false;
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/Lucene87RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/Lucene87RWCodec.java
deleted file mode 100644
index f7be2b69822..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/Lucene87RWCodec.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene87;
-
-import org.apache.lucene.backward_codecs.lucene50.Lucene50RWCompoundFormat;
-import org.apache.lucene.backward_codecs.lucene50.Lucene50RWTermVectorsFormat;
-import org.apache.lucene.backward_codecs.lucene80.Lucene80RWNormsFormat;
-import org.apache.lucene.backward_codecs.lucene84.Lucene84RWPostingsFormat;
-import org.apache.lucene.backward_codecs.lucene86.Lucene86RWPointsFormat;
-import org.apache.lucene.backward_codecs.lucene86.Lucene86RWSegmentInfoFormat;
-import org.apache.lucene.codecs.CompoundFormat;
-import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.PointsFormat;
-import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.SegmentInfoFormat;
-import org.apache.lucene.codecs.StoredFieldsFormat;
-import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
-
-/** RW impersonation of {@link Lucene87Codec}. */
-public class Lucene87RWCodec extends Lucene87Codec {
-
- private final PostingsFormat defaultPF = new Lucene84RWPostingsFormat();
- private final CompoundFormat compoundFormat = new Lucene50RWCompoundFormat();
- private final PostingsFormat postingsFormat =
- new PerFieldPostingsFormat() {
- @Override
- public PostingsFormat getPostingsFormatForField(String field) {
- return defaultPF;
- }
- };
- private final Mode mode;
-
- public Lucene87RWCodec() {
- this(Mode.BEST_SPEED);
- }
-
- public Lucene87RWCodec(Mode mode) {
- super(mode);
- this.mode = mode;
- }
-
- @Override
- public SegmentInfoFormat segmentInfoFormat() {
- return new Lucene86RWSegmentInfoFormat();
- }
-
- @Override
- public final CompoundFormat compoundFormat() {
- return compoundFormat;
- }
-
- @Override
- public NormsFormat normsFormat() {
- return new Lucene80RWNormsFormat();
- }
-
- @Override
- public PostingsFormat postingsFormat() {
- return postingsFormat;
- }
-
- @Override
- public TermVectorsFormat termVectorsFormat() {
- return new Lucene50RWTermVectorsFormat();
- }
-
- @Override
- public StoredFieldsFormat storedFieldsFormat() {
- return new Lucene87RWStoredFieldsFormat(mode.storedMode);
- }
-
- @Override
- public final PointsFormat pointsFormat() {
- return new Lucene86RWPointsFormat();
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/Lucene87RWStoredFieldsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/Lucene87RWStoredFieldsFormat.java
deleted file mode 100644
index 2f2b347ff44..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/Lucene87RWStoredFieldsFormat.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene87;
-
-import java.io.IOException;
-import org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50RWCompressingStoredFieldsFormat;
-import org.apache.lucene.codecs.StoredFieldsFormat;
-import org.apache.lucene.codecs.StoredFieldsWriter;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-
-/** RW impersonation of Lucene87StoredFieldsFormat. */
-public class Lucene87RWStoredFieldsFormat extends Lucene87StoredFieldsFormat {
-
- public Lucene87RWStoredFieldsFormat(Mode mode) {
- super(mode);
- }
-
- @Override
- public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context)
- throws IOException {
- String previous = si.putAttribute(MODE_KEY, mode.name());
- if (previous != null && previous.equals(mode.name()) == false) {
- throw new IllegalStateException(
- "found existing value for "
- + MODE_KEY
- + " for segment: "
- + si.name
- + "old="
- + previous
- + ", new="
- + mode.name());
- }
- return impl(mode).fieldsWriter(directory, si, context);
- }
-
- @Override
- StoredFieldsFormat impl(Mode mode) {
- switch (mode) {
- case BEST_SPEED:
- return new Lucene50RWCompressingStoredFieldsFormat(
- "Lucene87StoredFieldsFastData", BEST_SPEED_MODE, BEST_SPEED_BLOCK_LENGTH, 1024, 10);
- case BEST_COMPRESSION:
- return new Lucene50RWCompressingStoredFieldsFormat(
- "Lucene87StoredFieldsHighData",
- BEST_COMPRESSION_MODE,
- BEST_COMPRESSION_BLOCK_LENGTH,
- 4096,
- 10);
- default:
- throw new AssertionError();
- }
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormat.java
deleted file mode 100644
index c58727903f9..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormat.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene87;
-
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestLucene87StoredFieldsFormat extends BaseStoredFieldsFormatTestCase {
- @Override
- protected Codec getCodec() {
- return new Lucene87RWCodec();
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormatHighCompression.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormatHighCompression.java
deleted file mode 100644
index 43881f5a395..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormatHighCompression.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene87;
-
-import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.StoredField;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.StoredFields;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase;
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestLucene87StoredFieldsFormatHighCompression extends BaseStoredFieldsFormatTestCase {
- @Override
- protected Codec getCodec() {
- return new Lucene87RWCodec(Lucene87Codec.Mode.BEST_COMPRESSION);
- }
-
- /**
- * Change compression params (leaving it the same for old segments) and tests that nothing breaks.
- */
- public void testMixedCompressions() throws Exception {
- Directory dir = newDirectory();
- for (int i = 0; i < 10; i++) {
- IndexWriterConfig iwc = newIndexWriterConfig();
- iwc.setCodec(
- new Lucene87RWCodec(RandomPicks.randomFrom(random(), Lucene87Codec.Mode.values())));
- IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig());
- Document doc = new Document();
- doc.add(new StoredField("field1", "value1"));
- doc.add(new StoredField("field2", "value2"));
- iw.addDocument(doc);
- if (random().nextInt(4) == 0) {
- iw.forceMerge(1);
- }
- iw.commit();
- iw.close();
- }
-
- DirectoryReader ir = DirectoryReader.open(dir);
- assertEquals(10, ir.numDocs());
- StoredFields storedFields = ir.storedFields();
- for (int i = 0; i < 10; i++) {
- Document doc = storedFields.document(i);
- assertEquals("value1", doc.get("field1"));
- assertEquals("value2", doc.get("field2"));
- }
- ir.close();
- // checkindex
- dir.close();
- }
-
- public void testInvalidOptions() {
- expectThrows(
- NullPointerException.class,
- () -> {
- new Lucene87RWCodec(null);
- });
-
- expectThrows(
- NullPointerException.class,
- () -> {
- new Lucene87StoredFieldsFormat(null);
- });
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormatMergeInstance.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormatMergeInstance.java
deleted file mode 100644
index 7d7444cbed6..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormatMergeInstance.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene87;
-
-import org.apache.lucene.tests.util.LuceneTestCase.Nightly;
-
-/** Test the merge instance of the Lucene87 stored fields format. */
-@Nightly // N-2 formats are only tested on nightly runs
-public class TestLucene87StoredFieldsFormatMergeInstance extends TestLucene87StoredFieldsFormat {
-
- @Override
- protected boolean shouldTestMergeInstance() {
- return true;
- }
-}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestAncientIndicesCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestAncientIndicesCompatibility.java
index 246e6fe01b2..2fff52dd1b7 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestAncientIndicesCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestAncientIndicesCompatibility.java
@@ -199,7 +199,7 @@ public class TestAncientIndicesCompatibility extends LuceneTestCase {
checker.setInfoStream(new PrintStream(bos, false, UTF_8));
checker.setLevel(CheckIndex.Level.MIN_LEVEL_FOR_INTEGRITY_CHECKS);
CheckIndex.Status indexStatus = checker.checkIndex();
- if (version.startsWith("8.") || version.startsWith("9.")) {
+ if (version.startsWith("9.")) {
assertTrue(indexStatus.clean);
} else {
assertFalse(indexStatus.clean);
diff --git a/lucene/core/build.gradle b/lucene/core/build.gradle
index ae4f31261ae..68e240b6561 100644
--- a/lucene/core/build.gradle
+++ b/lucene/core/build.gradle
@@ -59,9 +59,6 @@ createForUtilRegenerateTask(project, "ForUtil", "ForUtil.java", "gen_ForUtil.py"
file("src/java/org/apache/lucene/codecs/lucene104"))
configure(project(":lucene:backward-codecs")) {
- createForUtilRegenerateTask(project, "ForUtil84", "ForUtil.java", "gen_ForUtil.py",
- file("src/java/org/apache/lucene/backward_codecs/lucene84"))
-
createForUtilRegenerateTask(project, "ForUtil90", "ForUtil.java", "gen_ForUtil.py",
file("src/java/org/apache/lucene/backward_codecs/lucene90"))