Apache Solr open-source search software
LUCENE-9705: Create Lucene90TermVectorsFormat (#2334)
Ignacio Veraf8be421ae1f4
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50TermVectorsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50TermVectorsFormat.javasimilarity index 97%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50TermVectorsFormat.javarename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50TermVectorsFormat.javaindex 270767ed589..1771e7fc89c 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50TermVectorsFormat.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/Lucene50TermVectorsFormat.java@@ -14,11 +14,11 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene50;+package org.apache.lucene.backward_codecs.lucene50;+import org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsFormat;import org.apache.lucene.codecs.CodecUtil;import org.apache.lucene.codecs.TermVectorsFormat;-import org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat;import org.apache.lucene.codecs.compressing.CompressionMode;import org.apache.lucene.codecs.compressing.FieldsIndexWriter;import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat;@@ -151,7 +151,7 @@ import org.apache.lucene.util.packed.PackedInts;** @lucene.experimental*/-public final class Lucene50TermVectorsFormat extends CompressingTermVectorsFormat {+public final class Lucene50TermVectorsFormat extends Lucene50CompressingTermVectorsFormat {/** Sole constructor. */public Lucene50TermVectorsFormat() {diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndex.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndex.javanew file mode 100644index 00000000000..59ec656e765--- /dev/null+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndex.java@@ -0,0 +1,33 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_codecs.lucene50.compressing;++import java.io.Closeable;+import java.io.IOException;+import org.apache.lucene.util.Accountable;++abstract class FieldsIndex implements Accountable, Cloneable, Closeable {++ /** Get the start pointer for the block that contains the given docID. */+ abstract long getStartPointer(int docID);++ /** Check the integrity of the index. */+ abstract void checkIntegrity() throws IOException;++ @Override+ public abstract FieldsIndex clone();+}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndexReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndexReader.javanew file mode 100644index 00000000000..05ff4ca5662--- /dev/null+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndexReader.java@@ -0,0 +1,155 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_codecs.lucene50.compressing;++import java.io.IOException;+import java.io.UncheckedIOException;+import java.util.Objects;+import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.index.IndexFileNames;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.RandomAccessInput;+import org.apache.lucene.util.RamUsageEstimator;+import org.apache.lucene.util.packed.DirectMonotonicReader;++final class FieldsIndexReader extends FieldsIndex {++ static final int VERSION_START = 0;+ static final int VERSION_CURRENT = 0;++ private static final long BASE_RAM_BYTES_USED =+ RamUsageEstimator.shallowSizeOfInstance(FieldsIndexReader.class);++ private final int maxDoc;+ private final int blockShift;+ private final int numChunks;+ private final DirectMonotonicReader.Meta docsMeta;+ private final DirectMonotonicReader.Meta startPointersMeta;+ private final IndexInput indexInput;+ private final long docsStartPointer,+ docsEndPointer,+ startPointersStartPointer,+ startPointersEndPointer;+ private final DirectMonotonicReader docs, startPointers;+ private final long maxPointer;++ FieldsIndexReader(+ Directory dir,+ String name,+ String suffix,+ String extension,+ String codecName,+ byte[] id,+ IndexInput metaIn)+ throws IOException {+ maxDoc = metaIn.readInt();+ blockShift = metaIn.readInt();+ numChunks = metaIn.readInt();+ docsStartPointer = metaIn.readLong();+ docsMeta = DirectMonotonicReader.loadMeta(metaIn, numChunks, blockShift);+ docsEndPointer = startPointersStartPointer = metaIn.readLong();+ startPointersMeta = DirectMonotonicReader.loadMeta(metaIn, numChunks, blockShift);+ startPointersEndPointer = metaIn.readLong();+ maxPointer = metaIn.readLong();++ indexInput =+ dir.openInput(IndexFileNames.segmentFileName(name, suffix, extension), IOContext.READ);+ boolean success = false;+ try {+ CodecUtil.checkIndexHeader(+ indexInput, codecName + "Idx", VERSION_START, VERSION_CURRENT, id, suffix);+ CodecUtil.retrieveChecksum(indexInput);+ success = true;+ } finally {+ if (success == false) {+ indexInput.close();+ }+ }+ final RandomAccessInput docsSlice =+ indexInput.randomAccessSlice(docsStartPointer, docsEndPointer - docsStartPointer);+ final RandomAccessInput startPointersSlice =+ indexInput.randomAccessSlice(+ startPointersStartPointer, startPointersEndPointer - startPointersStartPointer);+ docs = DirectMonotonicReader.getInstance(docsMeta, docsSlice);+ startPointers = DirectMonotonicReader.getInstance(startPointersMeta, startPointersSlice);+ }++ private FieldsIndexReader(FieldsIndexReader other) throws IOException {+ maxDoc = other.maxDoc;+ numChunks = other.numChunks;+ blockShift = other.blockShift;+ docsMeta = other.docsMeta;+ startPointersMeta = other.startPointersMeta;+ indexInput = other.indexInput.clone();+ docsStartPointer = other.docsStartPointer;+ docsEndPointer = other.docsEndPointer;+ startPointersStartPointer = other.startPointersStartPointer;+ startPointersEndPointer = other.startPointersEndPointer;+ maxPointer = other.maxPointer;+ final RandomAccessInput docsSlice =+ indexInput.randomAccessSlice(docsStartPointer, docsEndPointer - docsStartPointer);+ final RandomAccessInput startPointersSlice =+ indexInput.randomAccessSlice(+ startPointersStartPointer, startPointersEndPointer - startPointersStartPointer);+ docs = DirectMonotonicReader.getInstance(docsMeta, docsSlice);+ startPointers = DirectMonotonicReader.getInstance(startPointersMeta, startPointersSlice);+ }++ @Override+ public long ramBytesUsed() {+ return BASE_RAM_BYTES_USED+ + docsMeta.ramBytesUsed()+ + startPointersMeta.ramBytesUsed()+ + docs.ramBytesUsed()+ + startPointers.ramBytesUsed();+ }++ @Override+ public void close() throws IOException {+ indexInput.close();+ }++ @Override+ long getStartPointer(int docID) {+ Objects.checkIndex(docID, maxDoc);+ long blockIndex = docs.binarySearch(0, numChunks, docID);+ if (blockIndex < 0) {+ blockIndex = -2 - blockIndex;+ }+ return startPointers.get(blockIndex);+ }++ @Override+ public FieldsIndex clone() {+ try {+ return new FieldsIndexReader(this);+ } catch (IOException e) {+ throw new UncheckedIOException(e);+ }+ }++ public long getMaxPointer() {+ return maxPointer;+ }++ @Override+ void checkIntegrity() throws IOException {+ CodecUtil.checksumEntireFile(indexInput);+ }+}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/LegacyFieldsIndexReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/LegacyFieldsIndexReader.javanew file mode 100644index 00000000000..596ac825bd6--- /dev/null+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/LegacyFieldsIndexReader.java@@ -0,0 +1,233 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_codecs.lucene50.compressing;++import static org.apache.lucene.util.BitUtil.zigZagDecode;++import java.io.IOException;+import java.util.ArrayList;+import java.util.Collection;+import java.util.Collections;+import java.util.List;+import org.apache.lucene.index.CorruptIndexException;+import org.apache.lucene.index.SegmentInfo;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.util.Accountable;+import org.apache.lucene.util.Accountables;+import org.apache.lucene.util.ArrayUtil;+import org.apache.lucene.util.RamUsageEstimator;+import org.apache.lucene.util.packed.PackedInts;++final class LegacyFieldsIndexReader extends FieldsIndex {++ private static final long BASE_RAM_BYTES_USED =+ RamUsageEstimator.shallowSizeOfInstance(LegacyFieldsIndexReader.class);++ final int maxDoc;+ final int[] docBases;+ final long[] startPointers;+ final int[] avgChunkDocs;+ final long[] avgChunkSizes;+ final PackedInts.Reader[] docBasesDeltas; // delta from the avg+ final PackedInts.Reader[] startPointersDeltas; // delta from the avg++ // It is the responsibility of the caller to close fieldsIndexIn after this constructor+ // has been called+ LegacyFieldsIndexReader(IndexInput fieldsIndexIn, SegmentInfo si) throws IOException {+ maxDoc = si.maxDoc();+ int[] docBases = new int[16];+ long[] startPointers = new long[16];+ int[] avgChunkDocs = new int[16];+ long[] avgChunkSizes = new long[16];+ PackedInts.Reader[] docBasesDeltas = new PackedInts.Reader[16];+ PackedInts.Reader[] startPointersDeltas = new PackedInts.Reader[16];++ final int packedIntsVersion = fieldsIndexIn.readVInt();++ int blockCount = 0;++ for (; ; ) {+ final int numChunks = fieldsIndexIn.readVInt();+ if (numChunks == 0) {+ break;+ }+ if (blockCount == docBases.length) {+ final int newSize = ArrayUtil.oversize(blockCount + 1, 8);+ docBases = ArrayUtil.growExact(docBases, newSize);+ startPointers = ArrayUtil.growExact(startPointers, newSize);+ avgChunkDocs = ArrayUtil.growExact(avgChunkDocs, newSize);+ avgChunkSizes = ArrayUtil.growExact(avgChunkSizes, newSize);+ docBasesDeltas = ArrayUtil.growExact(docBasesDeltas, newSize);+ startPointersDeltas = ArrayUtil.growExact(startPointersDeltas, newSize);+ }++ // doc bases+ docBases[blockCount] = fieldsIndexIn.readVInt();+ avgChunkDocs[blockCount] = fieldsIndexIn.readVInt();+ final int bitsPerDocBase = fieldsIndexIn.readVInt();+ if (bitsPerDocBase > 32) {+ throw new CorruptIndexException(+ "Corrupted bitsPerDocBase: " + bitsPerDocBase, fieldsIndexIn);+ }+ docBasesDeltas[blockCount] =+ PackedInts.getReaderNoHeader(+ fieldsIndexIn,+ PackedInts.Format.PACKED,+ packedIntsVersion,+ numChunks,+ bitsPerDocBase);++ // start pointers+ startPointers[blockCount] = fieldsIndexIn.readVLong();+ avgChunkSizes[blockCount] = fieldsIndexIn.readVLong();+ final int bitsPerStartPointer = fieldsIndexIn.readVInt();+ if (bitsPerStartPointer > 64) {+ throw new CorruptIndexException(+ "Corrupted bitsPerStartPointer: " + bitsPerStartPointer, fieldsIndexIn);+ }+ startPointersDeltas[blockCount] =+ PackedInts.getReaderNoHeader(+ fieldsIndexIn,+ PackedInts.Format.PACKED,+ packedIntsVersion,+ numChunks,+ bitsPerStartPointer);++ ++blockCount;+ }++ this.docBases = ArrayUtil.copyOfSubArray(docBases, 0, blockCount);+ this.startPointers = ArrayUtil.copyOfSubArray(startPointers, 0, blockCount);+ this.avgChunkDocs = ArrayUtil.copyOfSubArray(avgChunkDocs, 0, blockCount);+ this.avgChunkSizes = ArrayUtil.copyOfSubArray(avgChunkSizes, 0, blockCount);+ this.docBasesDeltas = ArrayUtil.copyOfSubArray(docBasesDeltas, 0, blockCount);+ this.startPointersDeltas = ArrayUtil.copyOfSubArray(startPointersDeltas, 0, blockCount);+ }++ private int block(int docID) {+ int lo = 0, hi = docBases.length - 1;+ while (lo <= hi) {+ final int mid = (lo + hi) >>> 1;+ final int midValue = docBases[mid];+ if (midValue == docID) {+ return mid;+ } else if (midValue < docID) {+ lo = mid + 1;+ } else {+ hi = mid - 1;+ }+ }+ return hi;+ }++ private int relativeDocBase(int block, int relativeChunk) {+ final int expected = avgChunkDocs[block] * relativeChunk;+ final long delta = zigZagDecode(docBasesDeltas[block].get(relativeChunk));+ return expected + (int) delta;+ }++ private long relativeStartPointer(int block, int relativeChunk) {+ final long expected = avgChunkSizes[block] * relativeChunk;+ final long delta = zigZagDecode(startPointersDeltas[block].get(relativeChunk));+ return expected + delta;+ }++ private int relativeChunk(int block, int relativeDoc) {+ int lo = 0, hi = docBasesDeltas[block].size() - 1;+ while (lo <= hi) {+ final int mid = (lo + hi) >>> 1;+ final int midValue = relativeDocBase(block, mid);+ if (midValue == relativeDoc) {+ return mid;+ } else if (midValue < relativeDoc) {+ lo = mid + 1;+ } else {+ hi = mid - 1;+ }+ }+ return hi;+ }++ long getStartPointer(int docID) {+ if (docID < 0 || docID >= maxDoc) {+ throw new IllegalArgumentException("docID out of range [0-" + maxDoc + "]: " + docID);+ }+ final int block = block(docID);+ final int relativeChunk = relativeChunk(block, docID - docBases[block]);+ return startPointers[block] + relativeStartPointer(block, relativeChunk);+ }++ @Override+ public LegacyFieldsIndexReader clone() {+ return this;+ }++ @Override+ public long ramBytesUsed() {+ long res = BASE_RAM_BYTES_USED;++ res += RamUsageEstimator.shallowSizeOf(docBasesDeltas);+ for (PackedInts.Reader r : docBasesDeltas) {+ res += r.ramBytesUsed();+ }+ res += RamUsageEstimator.shallowSizeOf(startPointersDeltas);+ for (PackedInts.Reader r : startPointersDeltas) {+ res += r.ramBytesUsed();+ }++ res += RamUsageEstimator.sizeOf(docBases);+ res += RamUsageEstimator.sizeOf(startPointers);+ res += RamUsageEstimator.sizeOf(avgChunkDocs);+ res += RamUsageEstimator.sizeOf(avgChunkSizes);++ return res;+ }++ @Override+ public Collection<Accountable> getChildResources() {+ List<Accountable> resources = new ArrayList<>();++ long docBaseDeltaBytes = RamUsageEstimator.shallowSizeOf(docBasesDeltas);+ for (PackedInts.Reader r : docBasesDeltas) {+ docBaseDeltaBytes += r.ramBytesUsed();+ }+ resources.add(Accountables.namedAccountable("doc base deltas", docBaseDeltaBytes));++ long startPointerDeltaBytes = RamUsageEstimator.shallowSizeOf(startPointersDeltas);+ for (PackedInts.Reader r : startPointersDeltas) {+ startPointerDeltaBytes += r.ramBytesUsed();+ }+ resources.add(Accountables.namedAccountable("start pointer deltas", startPointerDeltaBytes));++ return Collections.unmodifiableList(resources);+ }++ @Override+ public String toString() {+ return getClass().getSimpleName() + "(blocks=" + docBases.length + ")";+ }++ @Override+ public void close() throws IOException {+ // nothing to do+ }++ @Override+ void checkIntegrity() throws IOException {+ // nothing to do, the index is checked at open time+ }+}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsFormat.javanew file mode 100644index 00000000000..628b21d7834--- /dev/null+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsFormat.java@@ -0,0 +1,118 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_codecs.lucene50.compressing;++import java.io.IOException;+import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.codecs.StoredFieldsFormat;+import org.apache.lucene.codecs.TermVectorsFormat;+import org.apache.lucene.codecs.TermVectorsReader;+import org.apache.lucene.codecs.TermVectorsWriter;+import org.apache.lucene.codecs.compressing.CompressionMode;+import org.apache.lucene.index.FieldInfos;+import org.apache.lucene.index.SegmentInfo;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.IOContext;++/**+ * A {@link TermVectorsFormat} that compresses chunks of documents together in order to improve the+ * compression ratio.+ *+ * @lucene.experimental+ */+public class Lucene50CompressingTermVectorsFormat extends TermVectorsFormat {++ /** format name */+ protected final String formatName;+ /** segment suffix */+ protected final String segmentSuffix;+ /** compression mode */+ protected final CompressionMode compressionMode;+ /** chunk size */+ protected final int chunkSize;+ /** block size */+ protected final int blockSize;++ /**+ * Create a new {@link Lucene50CompressingTermVectorsFormat}.+ *+ * <p><code>formatName</code> is the name of the format. This name will be used in the file+ * formats to perform {@link CodecUtil#checkIndexHeader codec header checks}.+ *+ * <p>The <code>compressionMode</code> parameter allows you to choose between compression+ * algorithms that have various compression and decompression speeds so that you can pick the one+ * that best fits your indexing and searching throughput. You should never instantiate two {@link+ * Lucene50CompressingTermVectorsFormat}s that have the same name but different {@link+ * CompressionMode}s.+ *+ * <p><code>chunkSize</code> is the minimum byte size of a chunk of documents. Higher values of+ * <code>chunkSize</code> should improve the compression ratio but will require more memory at+ * indexing time and might make document loading a little slower (depending on the size of your OS+ * cache compared to the size of your index).+ *+ * @param formatName the name of the {@link StoredFieldsFormat}+ * @param segmentSuffix a suffix to append to files created by this format+ * @param compressionMode the {@link CompressionMode} to use+ * @param chunkSize the minimum number of bytes of a single chunk of stored documents+ * @param blockSize the number of chunks to store in an index block.+ * @see CompressionMode+ */+ public Lucene50CompressingTermVectorsFormat(+ String formatName,+ String segmentSuffix,+ CompressionMode compressionMode,+ int chunkSize,+ int blockSize) {+ this.formatName = formatName;+ this.segmentSuffix = segmentSuffix;+ this.compressionMode = compressionMode;+ if (chunkSize < 1) {+ throw new IllegalArgumentException("chunkSize must be >= 1");+ }+ this.chunkSize = chunkSize;+ if (blockSize < 1) {+ throw new IllegalArgumentException("blockSize must be >= 1");+ }+ this.blockSize = blockSize;+ }++ @Override+ public final TermVectorsReader vectorsReader(+ Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context)+ throws IOException {+ return new Lucene50CompressingTermVectorsReader(+ directory, segmentInfo, segmentSuffix, fieldInfos, context, formatName, compressionMode);+ }++ @Override+ public TermVectorsWriter vectorsWriter(+ Directory directory, SegmentInfo segmentInfo, IOContext context) throws IOException {+ throw new UnsupportedOperationException("Old formats can't be used for writing");+ }++ @Override+ public String toString() {+ return getClass().getSimpleName()+ + "(compressionMode="+ + compressionMode+ + ", chunkSize="+ + chunkSize+ + ", blockSize="+ + blockSize+ + ")";+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsReader.javasimilarity index 95%rename from lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.javarename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsReader.javaindex a3230fa31e6..6d8a248042f 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsReader.java@@ -14,22 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.compressing;--import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.FLAGS_BITS;-import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.META_VERSION_START;-import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.OFFSETS;-import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.PACKED_BLOCK_SIZE;-import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.PAYLOADS;-import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.POSITIONS;-import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VECTORS_EXTENSION;-import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VECTORS_INDEX_CODEC_NAME;-import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VECTORS_INDEX_EXTENSION;-import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VECTORS_META_EXTENSION;-import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VERSION_CURRENT;-import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VERSION_META;-import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VERSION_OFFHEAP_INDEX;-import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VERSION_START;+package org.apache.lucene.backward_codecs.lucene50.compressing;import java.io.Closeable;import java.io.IOException;@@ -39,6 +24,8 @@ import java.util.Iterator;import java.util.NoSuchElementException;import org.apache.lucene.codecs.CodecUtil;import org.apache.lucene.codecs.TermVectorsReader;+import org.apache.lucene.codecs.compressing.CompressionMode;+import org.apache.lucene.codecs.compressing.Decompressor;import org.apache.lucene.index.BaseTermsEnum;import org.apache.lucene.index.CorruptIndexException;import org.apache.lucene.index.FieldInfo;@@ -67,11 +54,35 @@ import org.apache.lucene.util.packed.BlockPackedReaderIterator;import org.apache.lucene.util.packed.PackedInts;/**- * {@link TermVectorsReader} for {@link CompressingTermVectorsFormat}.+ * {@link TermVectorsReader} for {@link Lucene50CompressingTermVectorsFormat}.** @lucene.experimental*/-public final class CompressingTermVectorsReader extends TermVectorsReader implements Closeable {+public final class Lucene50CompressingTermVectorsReader extends TermVectorsReader+ implements Closeable {++ // hard limit on the maximum number of documents per chunk+ static final int MAX_DOCUMENTS_PER_CHUNK = 128;++ static final String VECTORS_EXTENSION = "tvd";+ static final String VECTORS_INDEX_EXTENSION = "tvx";+ static final String VECTORS_META_EXTENSION = "tvm";+ static final String VECTORS_INDEX_CODEC_NAME = "Lucene85TermVectorsIndex";++ static final int VERSION_START = 1;+ static final int VERSION_OFFHEAP_INDEX = 2;+ /** Version where all metadata were moved to the meta file. */+ static final int VERSION_META = 3;++ static final int VERSION_CURRENT = VERSION_META;+ static final int META_VERSION_START = 0;++ static final int PACKED_BLOCK_SIZE = 64;++ static final int POSITIONS = 0x01;+ static final int OFFSETS = 0x02;+ static final int PAYLOADS = 0x04;+ static final int FLAGS_BITS = PackedInts.bitsRequired(POSITIONS | OFFSETS | PAYLOADS);private final FieldInfos fieldInfos;final FieldsIndex indexReader;@@ -89,7 +100,7 @@ public final class CompressingTermVectorsReader extends TermVectorsReader implemprivate final long maxPointer; // end of the data section// used by clone- private CompressingTermVectorsReader(CompressingTermVectorsReader reader) {+ private Lucene50CompressingTermVectorsReader(Lucene50CompressingTermVectorsReader reader) {this.fieldInfos = reader.fieldInfos;this.vectorsStream = reader.vectorsStream.clone();this.indexReader = reader.indexReader.clone();@@ -108,7 +119,7 @@ public final class CompressingTermVectorsReader extends TermVectorsReader implem}/** Sole constructor. */- public CompressingTermVectorsReader(+ public Lucene50CompressingTermVectorsReader(Directory d,SegmentInfo si,String segmentSuffix,@@ -318,7 +329,7 @@ public final class CompressingTermVectorsReader extends TermVectorsReader implem@Overridepublic TermVectorsReader clone() {- return new CompressingTermVectorsReader(this);+ return new Lucene50CompressingTermVectorsReader(this);}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/package-info.javasimilarity index 81%rename from lucene/core/src/java/org/apache/lucene/codecs/lucene50/package-info.javarename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/package-info.javaindex af42de0ab68..53d5f4bd1af 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/package-info.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/package-info.java@@ -15,8 +15,5 @@* limitations under the License.*/-/**- * Components from the Lucene 5.0 index format See {@link org.apache.lucene.codecs.lucene90} for an- * overview of the index format.- */-package org.apache.lucene.codecs.lucene50;+/** Lucene 5.0 compressing format. */+package org.apache.lucene.backward_codecs.lucene50.compressing;diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene70/Lucene70Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene70/Lucene70Codec.javaindex e6e1d9e5e2d..4b7d8bfa7cd 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene70/Lucene70Codec.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene70/Lucene70Codec.java@@ -20,6 +20,7 @@ import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat;import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat.Mode;+import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat;import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;import org.apache.lucene.backward_codecs.lucene60.Lucene60PointsFormat;import org.apache.lucene.codecs.Codec;@@ -35,7 +36,6 @@ import org.apache.lucene.codecs.SegmentInfoFormat;import org.apache.lucene.codecs.StoredFieldsFormat;import org.apache.lucene.codecs.TermVectorsFormat;import org.apache.lucene.codecs.VectorFormat;-import org.apache.lucene.codecs.lucene50.Lucene50TermVectorsFormat;import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;@@ -87,7 +87,7 @@ public class Lucene70Codec extends Codec {}@Override- public final TermVectorsFormat termVectorsFormat() {+ public TermVectorsFormat termVectorsFormat() {return vectorsFormat;}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80Codec.javaindex 7136276da04..c7451211157 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80Codec.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80Codec.java@@ -19,6 +19,7 @@ package org.apache.lucene.backward_codecs.lucene80;import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat;+import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat;import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;import org.apache.lucene.backward_codecs.lucene60.Lucene60PointsFormat;import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat;@@ -34,7 +35,6 @@ import org.apache.lucene.codecs.SegmentInfoFormat;import org.apache.lucene.codecs.StoredFieldsFormat;import org.apache.lucene.codecs.TermVectorsFormat;import org.apache.lucene.codecs.VectorFormat;-import org.apache.lucene.codecs.lucene50.Lucene50TermVectorsFormat;import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;@@ -84,7 +84,7 @@ public class Lucene80Codec extends Codec {}@Override- public final TermVectorsFormat termVectorsFormat() {+ public TermVectorsFormat termVectorsFormat() {return vectorsFormat;}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84Codec.javaindex 81125f56075..57c0d412ee8 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84Codec.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene84/Lucene84Codec.java@@ -21,6 +21,7 @@ import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat;import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat.Mode;+import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat;import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;import org.apache.lucene.backward_codecs.lucene60.Lucene60PointsFormat;import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat;@@ -38,7 +39,6 @@ import org.apache.lucene.codecs.SegmentInfoFormat;import org.apache.lucene.codecs.StoredFieldsFormat;import org.apache.lucene.codecs.TermVectorsFormat;import org.apache.lucene.codecs.VectorFormat;-import org.apache.lucene.codecs.lucene50.Lucene50TermVectorsFormat;import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;@@ -99,7 +99,7 @@ public class Lucene84Codec extends Codec {}@Override- public final TermVectorsFormat termVectorsFormat() {+ public TermVectorsFormat termVectorsFormat() {return vectorsFormat;}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86Codec.javaindex 78fb85aec10..1cd5d4cc313 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86Codec.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene86/Lucene86Codec.java@@ -21,6 +21,7 @@ import java.util.Objects;import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat;+import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat;import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;import org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat;import org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat;@@ -37,7 +38,6 @@ import org.apache.lucene.codecs.SegmentInfoFormat;import org.apache.lucene.codecs.StoredFieldsFormat;import org.apache.lucene.codecs.TermVectorsFormat;import org.apache.lucene.codecs.VectorFormat;-import org.apache.lucene.codecs.lucene50.Lucene50TermVectorsFormat;import org.apache.lucene.codecs.lucene86.Lucene86PointsFormat;import org.apache.lucene.codecs.lucene86.Lucene86SegmentInfoFormat;import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;@@ -101,7 +101,7 @@ public class Lucene86Codec extends Codec {}@Override- public final TermVectorsFormat termVectorsFormat() {+ public TermVectorsFormat termVectorsFormat() {return vectorsFormat;}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/Lucene87Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/Lucene87Codec.javaindex 181e8fb912e..5703d5e5efa 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/Lucene87Codec.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/Lucene87Codec.java@@ -20,6 +20,7 @@ package org.apache.lucene.backward_codecs.lucene87;import java.util.Objects;import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat;import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat;+import org.apache.lucene.backward_codecs.lucene50.Lucene50TermVectorsFormat;import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat;import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat;import org.apache.lucene.backward_codecs.lucene80.Lucene80NormsFormat;@@ -37,7 +38,6 @@ import org.apache.lucene.codecs.SegmentInfoFormat;import org.apache.lucene.codecs.StoredFieldsFormat;import org.apache.lucene.codecs.TermVectorsFormat;import org.apache.lucene.codecs.VectorFormat;-import org.apache.lucene.codecs.lucene50.Lucene50TermVectorsFormat;import org.apache.lucene.codecs.lucene86.Lucene86PointsFormat;import org.apache.lucene.codecs.lucene86.Lucene86SegmentInfoFormat;import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat;@@ -113,7 +113,7 @@ public class Lucene87Codec extends Codec {}@Override- public final TermVectorsFormat termVectorsFormat() {+ public TermVectorsFormat termVectorsFormat() {return vectorsFormat;}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWTermVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWTermVectorsFormat.javanew file mode 100644index 00000000000..e3fef3fc9e0--- /dev/null+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/Lucene50RWTermVectorsFormat.java@@ -0,0 +1,29 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_codecs.lucene50;++import org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50RWCompressingTermVectorsFormat;+import org.apache.lucene.codecs.compressing.CompressionMode;++/** RW impersonation of Lucene50StoredFieldsFormat. */+public final class Lucene50RWTermVectorsFormat extends Lucene50RWCompressingTermVectorsFormat {++ /** Sole constructor. */+ public Lucene50RWTermVectorsFormat() {+ super("Lucene50TermVectorsData", "", CompressionMode.FAST, 1 << 12, 10);+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50TermVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50TermVectorsFormat.javasimilarity index 87%rename from lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50TermVectorsFormat.javarename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50TermVectorsFormat.javaindex b46f4f743e8..4c0f868f2f0 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50TermVectorsFormat.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/TestLucene50TermVectorsFormat.java@@ -14,15 +14,16 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.codecs.lucene50;+package org.apache.lucene.backward_codecs.lucene50;+import org.apache.lucene.backward_codecs.lucene87.Lucene87RWCodec;import org.apache.lucene.codecs.Codec;import org.apache.lucene.index.BaseTermVectorsFormatTestCase;-import org.apache.lucene.util.TestUtil;public class TestLucene50TermVectorsFormat extends BaseTermVectorsFormatTestCase {+@Overrideprotected Codec getCodec() {- return TestUtil.getDefaultCodec();+ return new Lucene87RWCodec();}}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndexWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndexWriter.javanew file mode 100644index 00000000000..c7ae5c1fcd5--- /dev/null+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/FieldsIndexWriter.java@@ -0,0 +1,202 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_codecs.lucene50.compressing;++import static org.apache.lucene.backward_codecs.lucene50.compressing.FieldsIndexReader.VERSION_CURRENT;++import java.io.Closeable;+import java.io.IOException;+import java.util.ArrayList;+import java.util.List;+import org.apache.lucene.codecs.Codec;+import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.index.CorruptIndexException;+import org.apache.lucene.index.IndexFileNames;+import org.apache.lucene.store.ChecksumIndexInput;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.util.IOUtils;+import org.apache.lucene.util.packed.DirectMonotonicReader;+import org.apache.lucene.util.packed.DirectMonotonicWriter;++/**+ * Efficient index format for block-based {@link Codec}s.+ *+ * <p>For each block of compressed stored fields, this stores the first document of the block and+ * the start pointer of the block in a {@link DirectMonotonicWriter}. At read time, the docID is+ * binary-searched in the {@link DirectMonotonicReader} that records doc IDS, and the returned index+ * is used to look up the start pointer in the {@link DirectMonotonicReader} that records start+ * pointers.+ *+ * @lucene.internal+ */+public final class FieldsIndexWriter implements Closeable {++ private final Directory dir;+ private final String name;+ private final String suffix;+ private final String extension;+ private final String codecName;+ private final byte[] id;+ private final int blockShift;+ private final IOContext ioContext;+ private IndexOutput docsOut;+ private IndexOutput filePointersOut;+ private int totalDocs;+ private int totalChunks;+ private long previousFP;++ FieldsIndexWriter(+ Directory dir,+ String name,+ String suffix,+ String extension,+ String codecName,+ byte[] id,+ int blockShift,+ IOContext ioContext)+ throws IOException {+ this.dir = dir;+ this.name = name;+ this.suffix = suffix;+ this.extension = extension;+ this.codecName = codecName;+ this.id = id;+ this.blockShift = blockShift;+ this.ioContext = ioContext;+ this.docsOut = dir.createTempOutput(name, codecName + "-doc_ids", ioContext);+ boolean success = false;+ try {+ CodecUtil.writeHeader(docsOut, codecName + "Docs", VERSION_CURRENT);+ filePointersOut = dir.createTempOutput(name, codecName + "file_pointers", ioContext);+ CodecUtil.writeHeader(filePointersOut, codecName + "FilePointers", VERSION_CURRENT);+ success = true;+ } finally {+ if (success == false) {+ close();+ }+ }+ }++ void writeIndex(int numDocs, long startPointer) throws IOException {+ assert startPointer >= previousFP;+ docsOut.writeVInt(numDocs);+ filePointersOut.writeVLong(startPointer - previousFP);+ previousFP = startPointer;+ totalDocs += numDocs;+ totalChunks++;+ }++ void finish(int numDocs, long maxPointer, IndexOutput metaOut) throws IOException {+ if (numDocs != totalDocs) {+ throw new IllegalStateException("Expected " + numDocs + " docs, but got " + totalDocs);+ }+ CodecUtil.writeFooter(docsOut);+ CodecUtil.writeFooter(filePointersOut);+ IOUtils.close(docsOut, filePointersOut);++ try (IndexOutput dataOut =+ dir.createOutput(IndexFileNames.segmentFileName(name, suffix, extension), ioContext)) {+ CodecUtil.writeIndexHeader(dataOut, codecName + "Idx", VERSION_CURRENT, id, suffix);++ metaOut.writeInt(numDocs);+ metaOut.writeInt(blockShift);+ metaOut.writeInt(totalChunks + 1);+ metaOut.writeLong(dataOut.getFilePointer());++ try (ChecksumIndexInput docsIn =+ dir.openChecksumInput(docsOut.getName(), IOContext.READONCE)) {+ CodecUtil.checkHeader(docsIn, codecName + "Docs", VERSION_CURRENT, VERSION_CURRENT);+ Throwable priorE = null;+ try {+ final DirectMonotonicWriter docs =+ DirectMonotonicWriter.getInstance(metaOut, dataOut, totalChunks + 1, blockShift);+ long doc = 0;+ docs.add(doc);+ for (int i = 0; i < totalChunks; ++i) {+ doc += docsIn.readVInt();+ docs.add(doc);+ }+ docs.finish();+ if (doc != totalDocs) {+ throw new CorruptIndexException("Docs don't add up", docsIn);+ }+ } catch (Throwable e) {+ priorE = e;+ } finally {+ CodecUtil.checkFooter(docsIn, priorE);+ }+ }+ dir.deleteFile(docsOut.getName());+ docsOut = null;++ metaOut.writeLong(dataOut.getFilePointer());+ try (ChecksumIndexInput filePointersIn =+ dir.openChecksumInput(filePointersOut.getName(), IOContext.READONCE)) {+ CodecUtil.checkHeader(+ filePointersIn, codecName + "FilePointers", VERSION_CURRENT, VERSION_CURRENT);+ Throwable priorE = null;+ try {+ final DirectMonotonicWriter filePointers =+ DirectMonotonicWriter.getInstance(metaOut, dataOut, totalChunks + 1, blockShift);+ long fp = 0;+ for (int i = 0; i < totalChunks; ++i) {+ fp += filePointersIn.readVLong();+ filePointers.add(fp);+ }+ if (maxPointer < fp) {+ throw new CorruptIndexException("File pointers don't add up", filePointersIn);+ }+ filePointers.add(maxPointer);+ filePointers.finish();+ } catch (Throwable e) {+ priorE = e;+ } finally {+ CodecUtil.checkFooter(filePointersIn, priorE);+ }+ }+ dir.deleteFile(filePointersOut.getName());+ filePointersOut = null;++ metaOut.writeLong(dataOut.getFilePointer());+ metaOut.writeLong(maxPointer);++ CodecUtil.writeFooter(dataOut);+ }+ }++ @Override+ public void close() throws IOException {+ try {+ IOUtils.close(docsOut, filePointersOut);+ } finally {+ List<String> fileNames = new ArrayList<>();+ if (docsOut != null) {+ fileNames.add(docsOut.getName());+ }+ if (filePointersOut != null) {+ fileNames.add(filePointersOut.getName());+ }+ try {+ IOUtils.deleteFiles(dir, fileNames);+ } finally {+ docsOut = filePointersOut = null;+ }+ }+ }+}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsWriter.javanew file mode 100644index 00000000000..926a4be2c53--- /dev/null+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingTermVectorsWriter.java@@ -0,0 +1,953 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_codecs.lucene50.compressing;++import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.FLAGS_BITS;+import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.MAX_DOCUMENTS_PER_CHUNK;+import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.OFFSETS;+import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.PACKED_BLOCK_SIZE;+import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.PAYLOADS;+import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.POSITIONS;+import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.VECTORS_EXTENSION;+import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.VECTORS_INDEX_CODEC_NAME;+import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.VECTORS_INDEX_EXTENSION;+import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.VECTORS_META_EXTENSION;+import static org.apache.lucene.backward_codecs.lucene50.compressing.Lucene50CompressingTermVectorsReader.VERSION_CURRENT;++import java.io.IOException;+import java.util.ArrayDeque;+import java.util.Arrays;+import java.util.Collection;+import java.util.Deque;+import java.util.Iterator;+import java.util.List;+import java.util.SortedSet;+import java.util.TreeSet;+import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.codecs.TermVectorsReader;+import org.apache.lucene.codecs.TermVectorsWriter;+import org.apache.lucene.codecs.compressing.CompressionMode;+import org.apache.lucene.codecs.compressing.Compressor;+import org.apache.lucene.codecs.compressing.MatchingReaders;+import org.apache.lucene.index.CorruptIndexException;+import org.apache.lucene.index.FieldInfo;+import org.apache.lucene.index.FieldInfos;+import org.apache.lucene.index.Fields;+import org.apache.lucene.index.IndexFileNames;+import org.apache.lucene.index.MergeState;+import org.apache.lucene.index.SegmentInfo;+import org.apache.lucene.store.ByteBuffersDataOutput;+import org.apache.lucene.store.DataInput;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.util.Accountable;+import org.apache.lucene.util.ArrayUtil;+import org.apache.lucene.util.Bits;+import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.IOUtils;+import org.apache.lucene.util.StringHelper;+import org.apache.lucene.util.packed.BlockPackedWriter;+import org.apache.lucene.util.packed.PackedInts;++/**+ * {@link TermVectorsWriter} for {@link Lucene50CompressingTermVectorsFormat}.+ *+ * @lucene.experimental+ */+public final class Lucene50CompressingTermVectorsWriter extends TermVectorsWriter {++ private final String segment;+ private FieldsIndexWriter indexWriter;+ private IndexOutput metaStream, vectorsStream;++ private final CompressionMode compressionMode;+ private final Compressor compressor;+ private final int chunkSize;++ private long numDirtyChunks; // number of incomplete compressed blocks written+ private long numDirtyDocs; // cumulative number of missing docs in incomplete chunks++ /** a pending doc */+ private class DocData {+ final int numFields;+ final Deque<FieldData> fields;+ final int posStart, offStart, payStart;++ DocData(int numFields, int posStart, int offStart, int payStart) {+ this.numFields = numFields;+ this.fields = new ArrayDeque<>(numFields);+ this.posStart = posStart;+ this.offStart = offStart;+ this.payStart = payStart;+ }++ FieldData addField(+ int fieldNum, int numTerms, boolean positions, boolean offsets, boolean payloads) {+ final FieldData field;+ if (fields.isEmpty()) {+ field =+ new FieldData(+ fieldNum, numTerms, positions, offsets, payloads, posStart, offStart, payStart);+ } else {+ final FieldData last = fields.getLast();+ final int posStart = last.posStart + (last.hasPositions ? last.totalPositions : 0);+ final int offStart = last.offStart + (last.hasOffsets ? last.totalPositions : 0);+ final int payStart = last.payStart + (last.hasPayloads ? last.totalPositions : 0);+ field =+ new FieldData(+ fieldNum, numTerms, positions, offsets, payloads, posStart, offStart, payStart);+ }+ fields.add(field);+ return field;+ }+ }++ private DocData addDocData(int numVectorFields) {+ FieldData last = null;+ for (Iterator<DocData> it = pendingDocs.descendingIterator(); it.hasNext(); ) {+ final DocData doc = it.next();+ if (!doc.fields.isEmpty()) {+ last = doc.fields.getLast();+ break;+ }+ }+ final DocData doc;+ if (last == null) {+ doc = new DocData(numVectorFields, 0, 0, 0);+ } else {+ final int posStart = last.posStart + (last.hasPositions ? last.totalPositions : 0);+ final int offStart = last.offStart + (last.hasOffsets ? last.totalPositions : 0);+ final int payStart = last.payStart + (last.hasPayloads ? last.totalPositions : 0);+ doc = new DocData(numVectorFields, posStart, offStart, payStart);+ }+ pendingDocs.add(doc);+ return doc;+ }++ /** a pending field */+ private class FieldData {+ final boolean hasPositions, hasOffsets, hasPayloads;+ final int fieldNum, flags, numTerms;+ final int[] freqs, prefixLengths, suffixLengths;+ final int posStart, offStart, payStart;+ int totalPositions;+ int ord;++ FieldData(+ int fieldNum,+ int numTerms,+ boolean positions,+ boolean offsets,+ boolean payloads,+ int posStart,+ int offStart,+ int payStart) {+ this.fieldNum = fieldNum;+ this.numTerms = numTerms;+ this.hasPositions = positions;+ this.hasOffsets = offsets;+ this.hasPayloads = payloads;+ this.flags =+ (positions ? POSITIONS : 0) | (offsets ? OFFSETS : 0) | (payloads ? PAYLOADS : 0);+ this.freqs = new int[numTerms];+ this.prefixLengths = new int[numTerms];+ this.suffixLengths = new int[numTerms];+ this.posStart = posStart;+ this.offStart = offStart;+ this.payStart = payStart;+ totalPositions = 0;+ ord = 0;+ }++ void addTerm(int freq, int prefixLength, int suffixLength) {+ freqs[ord] = freq;+ prefixLengths[ord] = prefixLength;+ suffixLengths[ord] = suffixLength;+ ++ord;+ }++ void addPosition(int position, int startOffset, int length, int payloadLength) {+ if (hasPositions) {+ if (posStart + totalPositions == positionsBuf.length) {+ positionsBuf = ArrayUtil.grow(positionsBuf);+ }+ positionsBuf[posStart + totalPositions] = position;+ }+ if (hasOffsets) {+ if (offStart + totalPositions == startOffsetsBuf.length) {+ final int newLength = ArrayUtil.oversize(offStart + totalPositions, 4);+ startOffsetsBuf = ArrayUtil.growExact(startOffsetsBuf, newLength);+ lengthsBuf = ArrayUtil.growExact(lengthsBuf, newLength);+ }+ startOffsetsBuf[offStart + totalPositions] = startOffset;+ lengthsBuf[offStart + totalPositions] = length;+ }+ if (hasPayloads) {+ if (payStart + totalPositions == payloadLengthsBuf.length) {+ payloadLengthsBuf = ArrayUtil.grow(payloadLengthsBuf);+ }+ payloadLengthsBuf[payStart + totalPositions] = payloadLength;+ }+ ++totalPositions;+ }+ }++ private int numDocs; // total number of docs seen+ private final Deque<DocData> pendingDocs; // pending docs+ private DocData curDoc; // current document+ private FieldData curField; // current field+ private final BytesRef lastTerm;+ private int[] positionsBuf, startOffsetsBuf, lengthsBuf, payloadLengthsBuf;+ private final ByteBuffersDataOutput termSuffixes; // buffered term suffixes+ private final ByteBuffersDataOutput payloadBytes; // buffered term payloads+ private final BlockPackedWriter writer;++ /** Sole constructor. */+ Lucene50CompressingTermVectorsWriter(+ Directory directory,+ SegmentInfo si,+ String segmentSuffix,+ IOContext context,+ String formatName,+ CompressionMode compressionMode,+ int chunkSize,+ int blockShift)+ throws IOException {+ assert directory != null;+ this.segment = si.name;+ this.compressionMode = compressionMode;+ this.compressor = compressionMode.newCompressor();+ this.chunkSize = chunkSize;++ numDocs = 0;+ pendingDocs = new ArrayDeque<>();+ termSuffixes = ByteBuffersDataOutput.newResettableInstance();+ payloadBytes = ByteBuffersDataOutput.newResettableInstance();+ lastTerm = new BytesRef(ArrayUtil.oversize(30, 1));++ boolean success = false;+ try {+ metaStream =+ directory.createOutput(+ IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_META_EXTENSION),+ context);+ CodecUtil.writeIndexHeader(+ metaStream,+ VECTORS_INDEX_CODEC_NAME + "Meta",+ VERSION_CURRENT,+ si.getId(),+ segmentSuffix);+ assert CodecUtil.indexHeaderLength(VECTORS_INDEX_CODEC_NAME + "Meta", segmentSuffix)+ == metaStream.getFilePointer();++ vectorsStream =+ directory.createOutput(+ IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION), context);+ CodecUtil.writeIndexHeader(+ vectorsStream, formatName, VERSION_CURRENT, si.getId(), segmentSuffix);+ assert CodecUtil.indexHeaderLength(formatName, segmentSuffix)+ == vectorsStream.getFilePointer();++ indexWriter =+ new FieldsIndexWriter(+ directory,+ segment,+ segmentSuffix,+ VECTORS_INDEX_EXTENSION,+ VECTORS_INDEX_CODEC_NAME,+ si.getId(),+ blockShift,+ context);++ metaStream.writeVInt(PackedInts.VERSION_CURRENT);+ metaStream.writeVInt(chunkSize);+ writer = new BlockPackedWriter(vectorsStream, PACKED_BLOCK_SIZE);++ positionsBuf = new int[1024];+ startOffsetsBuf = new int[1024];+ lengthsBuf = new int[1024];+ payloadLengthsBuf = new int[1024];++ success = true;+ } finally {+ if (!success) {+ IOUtils.closeWhileHandlingException(metaStream, vectorsStream, indexWriter, indexWriter);+ }+ }+ }++ @Override+ public void close() throws IOException {+ try {+ IOUtils.close(metaStream, vectorsStream, indexWriter);+ } finally {+ metaStream = null;+ vectorsStream = null;+ indexWriter = null;+ }+ }++ @Override+ public void startDocument(int numVectorFields) throws IOException {+ curDoc = addDocData(numVectorFields);+ }++ @Override+ public void finishDocument() throws IOException {+ // append the payload bytes of the doc after its terms+ payloadBytes.copyTo(termSuffixes);+ payloadBytes.reset();+ ++numDocs;+ if (triggerFlush()) {+ flush();+ }+ curDoc = null;+ }++ @Override+ public void startField(+ FieldInfo info, int numTerms, boolean positions, boolean offsets, boolean payloads)+ throws IOException {+ curField = curDoc.addField(info.number, numTerms, positions, offsets, payloads);+ lastTerm.length = 0;+ }++ @Override+ public void finishField() throws IOException {+ curField = null;+ }++ @Override+ public void startTerm(BytesRef term, int freq) throws IOException {+ assert freq >= 1;+ final int prefix;+ if (lastTerm.length == 0) {+ // no previous term: no bytes to write+ prefix = 0;+ } else {+ prefix = StringHelper.bytesDifference(lastTerm, term);+ }+ curField.addTerm(freq, prefix, term.length - prefix);+ termSuffixes.writeBytes(term.bytes, term.offset + prefix, term.length - prefix);+ // copy last term+ if (lastTerm.bytes.length < term.length) {+ lastTerm.bytes = new byte[ArrayUtil.oversize(term.length, 1)];+ }+ lastTerm.offset = 0;+ lastTerm.length = term.length;+ System.arraycopy(term.bytes, term.offset, lastTerm.bytes, 0, term.length);+ }++ @Override+ public void addPosition(int position, int startOffset, int endOffset, BytesRef payload)+ throws IOException {+ assert curField.flags != 0;+ curField.addPosition(+ position, startOffset, endOffset - startOffset, payload == null ? 0 : payload.length);+ if (curField.hasPayloads && payload != null) {+ payloadBytes.writeBytes(payload.bytes, payload.offset, payload.length);+ }+ }++ private boolean triggerFlush() {+ return termSuffixes.size() >= chunkSize || pendingDocs.size() >= MAX_DOCUMENTS_PER_CHUNK;+ }++ private void flush() throws IOException {+ final int chunkDocs = pendingDocs.size();+ assert chunkDocs > 0 : chunkDocs;++ // write the index file+ indexWriter.writeIndex(chunkDocs, vectorsStream.getFilePointer());++ final int docBase = numDocs - chunkDocs;+ vectorsStream.writeVInt(docBase);+ vectorsStream.writeVInt(chunkDocs);++ // total number of fields of the chunk+ final int totalFields = flushNumFields(chunkDocs);++ if (totalFields > 0) {+ // unique field numbers (sorted)+ final int[] fieldNums = flushFieldNums();+ // offsets in the array of unique field numbers+ flushFields(totalFields, fieldNums);+ // flags (does the field have positions, offsets, payloads?)+ flushFlags(totalFields, fieldNums);+ // number of terms of each field+ flushNumTerms(totalFields);+ // prefix and suffix lengths for each field+ flushTermLengths();+ // term freqs - 1 (because termFreq is always >=1) for each term+ flushTermFreqs();+ // positions for all terms, when enabled+ flushPositions();+ // offsets for all terms, when enabled+ flushOffsets(fieldNums);+ // payload lengths for all terms, when enabled+ flushPayloadLengths();++ // compress terms and payloads and write them to the output+ //+ // TODO: We could compress in the slices we already have in the buffer (min/max slice+ // can be set on the buffer itself).+ byte[] content = termSuffixes.toArrayCopy();+ compressor.compress(content, 0, content.length, vectorsStream);+ }++ // reset+ pendingDocs.clear();+ curDoc = null;+ curField = null;+ termSuffixes.reset();+ }++ private int flushNumFields(int chunkDocs) throws IOException {+ if (chunkDocs == 1) {+ final int numFields = pendingDocs.getFirst().numFields;+ vectorsStream.writeVInt(numFields);+ return numFields;+ } else {+ writer.reset(vectorsStream);+ int totalFields = 0;+ for (DocData dd : pendingDocs) {+ writer.add(dd.numFields);+ totalFields += dd.numFields;+ }+ writer.finish();+ return totalFields;+ }+ }++ /** Returns a sorted array containing unique field numbers */+ private int[] flushFieldNums() throws IOException {+ SortedSet<Integer> fieldNums = new TreeSet<>();+ for (DocData dd : pendingDocs) {+ for (FieldData fd : dd.fields) {+ fieldNums.add(fd.fieldNum);+ }+ }++ final int numDistinctFields = fieldNums.size();+ assert numDistinctFields > 0;+ final int bitsRequired = PackedInts.bitsRequired(fieldNums.last());+ final int token = (Math.min(numDistinctFields - 1, 0x07) << 5) | bitsRequired;+ vectorsStream.writeByte((byte) token);+ if (numDistinctFields - 1 >= 0x07) {+ vectorsStream.writeVInt(numDistinctFields - 1 - 0x07);+ }+ final PackedInts.Writer writer =+ PackedInts.getWriterNoHeader(+ vectorsStream, PackedInts.Format.PACKED, fieldNums.size(), bitsRequired, 1);+ for (Integer fieldNum : fieldNums) {+ writer.add(fieldNum);+ }+ writer.finish();++ int[] fns = new int[fieldNums.size()];+ int i = 0;+ for (Integer key : fieldNums) {+ fns[i++] = key;+ }+ return fns;+ }++ private void flushFields(int totalFields, int[] fieldNums) throws IOException {+ final PackedInts.Writer writer =+ PackedInts.getWriterNoHeader(+ vectorsStream,+ PackedInts.Format.PACKED,+ totalFields,+ PackedInts.bitsRequired(fieldNums.length - 1),+ 1);+ for (DocData dd : pendingDocs) {+ for (FieldData fd : dd.fields) {+ final int fieldNumIndex = Arrays.binarySearch(fieldNums, fd.fieldNum);+ assert fieldNumIndex >= 0;+ writer.add(fieldNumIndex);+ }+ }+ writer.finish();+ }++ private void flushFlags(int totalFields, int[] fieldNums) throws IOException {+ // check if fields always have the same flags+ boolean nonChangingFlags = true;+ int[] fieldFlags = new int[fieldNums.length];+ Arrays.fill(fieldFlags, -1);+ outer:+ for (DocData dd : pendingDocs) {+ for (FieldData fd : dd.fields) {+ final int fieldNumOff = Arrays.binarySearch(fieldNums, fd.fieldNum);+ assert fieldNumOff >= 0;+ if (fieldFlags[fieldNumOff] == -1) {+ fieldFlags[fieldNumOff] = fd.flags;+ } else if (fieldFlags[fieldNumOff] != fd.flags) {+ nonChangingFlags = false;+ break outer;+ }+ }+ }++ if (nonChangingFlags) {+ // write one flag per field num+ vectorsStream.writeVInt(0);+ final PackedInts.Writer writer =+ PackedInts.getWriterNoHeader(+ vectorsStream, PackedInts.Format.PACKED, fieldFlags.length, FLAGS_BITS, 1);+ for (int flags : fieldFlags) {+ assert flags >= 0;+ writer.add(flags);+ }+ assert writer.ord() == fieldFlags.length - 1;+ writer.finish();+ } else {+ // write one flag for every field instance+ vectorsStream.writeVInt(1);+ final PackedInts.Writer writer =+ PackedInts.getWriterNoHeader(+ vectorsStream, PackedInts.Format.PACKED, totalFields, FLAGS_BITS, 1);+ for (DocData dd : pendingDocs) {+ for (FieldData fd : dd.fields) {+ writer.add(fd.flags);+ }+ }+ assert writer.ord() == totalFields - 1;+ writer.finish();+ }+ }++ private void flushNumTerms(int totalFields) throws IOException {+ int maxNumTerms = 0;+ for (DocData dd : pendingDocs) {+ for (FieldData fd : dd.fields) {+ maxNumTerms |= fd.numTerms;+ }+ }+ final int bitsRequired = PackedInts.bitsRequired(maxNumTerms);+ vectorsStream.writeVInt(bitsRequired);+ final PackedInts.Writer writer =+ PackedInts.getWriterNoHeader(+ vectorsStream, PackedInts.Format.PACKED, totalFields, bitsRequired, 1);+ for (DocData dd : pendingDocs) {+ for (FieldData fd : dd.fields) {+ writer.add(fd.numTerms);+ }+ }+ assert writer.ord() == totalFields - 1;+ writer.finish();+ }++ private void flushTermLengths() throws IOException {+ writer.reset(vectorsStream);+ for (DocData dd : pendingDocs) {+ for (FieldData fd : dd.fields) {+ for (int i = 0; i < fd.numTerms; ++i) {+ writer.add(fd.prefixLengths[i]);+ }+ }+ }+ writer.finish();+ writer.reset(vectorsStream);+ for (DocData dd : pendingDocs) {+ for (FieldData fd : dd.fields) {+ for (int i = 0; i < fd.numTerms; ++i) {+ writer.add(fd.suffixLengths[i]);+ }+ }+ }+ writer.finish();+ }++ private void flushTermFreqs() throws IOException {+ writer.reset(vectorsStream);+ for (DocData dd : pendingDocs) {+ for (FieldData fd : dd.fields) {+ for (int i = 0; i < fd.numTerms; ++i) {+ writer.add(fd.freqs[i] - 1);+ }+ }+ }+ writer.finish();+ }++ private void flushPositions() throws IOException {+ writer.reset(vectorsStream);+ for (DocData dd : pendingDocs) {+ for (FieldData fd : dd.fields) {+ if (fd.hasPositions) {+ int pos = 0;+ for (int i = 0; i < fd.numTerms; ++i) {+ int previousPosition = 0;+ for (int j = 0; j < fd.freqs[i]; ++j) {+ final int position = positionsBuf[fd.posStart + pos++];+ writer.add(position - previousPosition);+ previousPosition = position;+ }+ }+ assert pos == fd.totalPositions;+ }+ }+ }+ writer.finish();+ }++ private void flushOffsets(int[] fieldNums) throws IOException {+ boolean hasOffsets = false;+ long[] sumPos = new long[fieldNums.length];+ long[] sumOffsets = new long[fieldNums.length];+ for (DocData dd : pendingDocs) {+ for (FieldData fd : dd.fields) {+ hasOffsets |= fd.hasOffsets;+ if (fd.hasOffsets && fd.hasPositions) {+ final int fieldNumOff = Arrays.binarySearch(fieldNums, fd.fieldNum);+ int pos = 0;+ for (int i = 0; i < fd.numTerms; ++i) {+ sumPos[fieldNumOff] += positionsBuf[fd.posStart + fd.freqs[i] - 1 + pos];+ sumOffsets[fieldNumOff] += startOffsetsBuf[fd.offStart + fd.freqs[i] - 1 + pos];+ pos += fd.freqs[i];+ }+ assert pos == fd.totalPositions;+ }+ }+ }++ if (!hasOffsets) {+ // nothing to do+ return;+ }++ final float[] charsPerTerm = new float[fieldNums.length];+ for (int i = 0; i < fieldNums.length; ++i) {+ charsPerTerm[i] =+ (sumPos[i] <= 0 || sumOffsets[i] <= 0) ? 0 : (float) ((double) sumOffsets[i] / sumPos[i]);+ }++ // start offsets+ for (int i = 0; i < fieldNums.length; ++i) {+ vectorsStream.writeInt(Float.floatToRawIntBits(charsPerTerm[i]));+ }++ writer.reset(vectorsStream);+ for (DocData dd : pendingDocs) {+ for (FieldData fd : dd.fields) {+ if ((fd.flags & OFFSETS) != 0) {+ final int fieldNumOff = Arrays.binarySearch(fieldNums, fd.fieldNum);+ final float cpt = charsPerTerm[fieldNumOff];+ int pos = 0;+ for (int i = 0; i < fd.numTerms; ++i) {+ int previousPos = 0;+ int previousOff = 0;+ for (int j = 0; j < fd.freqs[i]; ++j) {+ final int position = fd.hasPositions ? positionsBuf[fd.posStart + pos] : 0;+ final int startOffset = startOffsetsBuf[fd.offStart + pos];+ writer.add(startOffset - previousOff - (int) (cpt * (position - previousPos)));+ previousPos = position;+ previousOff = startOffset;+ ++pos;+ }+ }+ }+ }+ }+ writer.finish();++ // lengths+ writer.reset(vectorsStream);+ for (DocData dd : pendingDocs) {+ for (FieldData fd : dd.fields) {+ if ((fd.flags & OFFSETS) != 0) {+ int pos = 0;+ for (int i = 0; i < fd.numTerms; ++i) {+ for (int j = 0; j < fd.freqs[i]; ++j) {+ writer.add(+ lengthsBuf[fd.offStart + pos++] - fd.prefixLengths[i] - fd.suffixLengths[i]);+ }+ }+ assert pos == fd.totalPositions;+ }+ }+ }+ writer.finish();+ }++ private void flushPayloadLengths() throws IOException {+ writer.reset(vectorsStream);+ for (DocData dd : pendingDocs) {+ for (FieldData fd : dd.fields) {+ if (fd.hasPayloads) {+ for (int i = 0; i < fd.totalPositions; ++i) {+ writer.add(payloadLengthsBuf[fd.payStart + i]);+ }+ }+ }+ }+ writer.finish();+ }++ @Override+ public void finish(FieldInfos fis, int numDocs) throws IOException {+ if (!pendingDocs.isEmpty()) {+ numDirtyChunks++; // incomplete: we had to force this flush+ final long expectedChunkDocs =+ Math.min(+ MAX_DOCUMENTS_PER_CHUNK,+ (long) ((double) chunkSize / termSuffixes.size() * pendingDocs.size()));+ numDirtyDocs += expectedChunkDocs - pendingDocs.size();+ flush();+ }+ if (numDocs != this.numDocs) {+ throw new RuntimeException(+ "Wrote " + this.numDocs + " docs, finish called with numDocs=" + numDocs);+ }+ indexWriter.finish(numDocs, vectorsStream.getFilePointer(), metaStream);+ metaStream.writeVLong(numDirtyChunks);+ metaStream.writeVLong(numDirtyDocs);+ CodecUtil.writeFooter(metaStream);+ CodecUtil.writeFooter(vectorsStream);+ }++ @Override+ public void addProx(int numProx, DataInput positions, DataInput offsets) throws IOException {+ assert (curField.hasPositions) == (positions != null);+ assert (curField.hasOffsets) == (offsets != null);++ if (curField.hasPositions) {+ final int posStart = curField.posStart + curField.totalPositions;+ if (posStart + numProx > positionsBuf.length) {+ positionsBuf = ArrayUtil.grow(positionsBuf, posStart + numProx);+ }+ int position = 0;+ if (curField.hasPayloads) {+ final int payStart = curField.payStart + curField.totalPositions;+ if (payStart + numProx > payloadLengthsBuf.length) {+ payloadLengthsBuf = ArrayUtil.grow(payloadLengthsBuf, payStart + numProx);+ }+ for (int i = 0; i < numProx; ++i) {+ final int code = positions.readVInt();+ if ((code & 1) != 0) {+ // This position has a payload+ final int payloadLength = positions.readVInt();+ payloadLengthsBuf[payStart + i] = payloadLength;+ payloadBytes.copyBytes(positions, payloadLength);+ } else {+ payloadLengthsBuf[payStart + i] = 0;+ }+ position += code >>> 1;+ positionsBuf[posStart + i] = position;+ }+ } else {+ for (int i = 0; i < numProx; ++i) {+ position += (positions.readVInt() >>> 1);+ positionsBuf[posStart + i] = position;+ }+ }+ }++ if (curField.hasOffsets) {+ final int offStart = curField.offStart + curField.totalPositions;+ if (offStart + numProx > startOffsetsBuf.length) {+ final int newLength = ArrayUtil.oversize(offStart + numProx, 4);+ startOffsetsBuf = ArrayUtil.growExact(startOffsetsBuf, newLength);+ lengthsBuf = ArrayUtil.growExact(lengthsBuf, newLength);+ }+ int lastOffset = 0, startOffset, endOffset;+ for (int i = 0; i < numProx; ++i) {+ startOffset = lastOffset + offsets.readVInt();+ endOffset = startOffset + offsets.readVInt();+ lastOffset = endOffset;+ startOffsetsBuf[offStart + i] = startOffset;+ lengthsBuf[offStart + i] = endOffset - startOffset;+ }+ }++ curField.totalPositions += numProx;+ }++ // bulk merge is scary: its caused corruption bugs in the past.+ // we try to be extra safe with this impl, but add an escape hatch to+ // have a workaround for undiscovered bugs.+ static final String BULK_MERGE_ENABLED_SYSPROP =+ Lucene50CompressingTermVectorsWriter.class.getName() + ".enableBulkMerge";+ static final boolean BULK_MERGE_ENABLED;++ static {+ boolean v = true;+ try {+ v = Boolean.parseBoolean(System.getProperty(BULK_MERGE_ENABLED_SYSPROP, "true"));+ } catch (SecurityException ignored) {+ }+ BULK_MERGE_ENABLED = v;+ }++ @Override+ public int merge(MergeState mergeState) throws IOException {+ if (mergeState.needsIndexSort) {+ // TODO: can we gain back some optos even if index is sorted? E.g. if sort results in large+ // chunks of contiguous docs from one sub+ // being copied over...?+ return super.merge(mergeState);+ }+ int docCount = 0;+ int numReaders = mergeState.maxDocs.length;++ MatchingReaders matching = new MatchingReaders(mergeState);++ for (int readerIndex = 0; readerIndex < numReaders; readerIndex++) {+ Lucene50CompressingTermVectorsReader matchingVectorsReader = null;+ final TermVectorsReader vectorsReader = mergeState.termVectorsReaders[readerIndex];+ if (matching.matchingReaders[readerIndex]) {+ // we can only bulk-copy if the matching reader is also a CompressingTermVectorsReader+ if (vectorsReader != null+ && vectorsReader instanceof Lucene50CompressingTermVectorsReader) {+ matchingVectorsReader = (Lucene50CompressingTermVectorsReader) vectorsReader;+ }+ }++ final int maxDoc = mergeState.maxDocs[readerIndex];+ final Bits liveDocs = mergeState.liveDocs[readerIndex];++ if (matchingVectorsReader != null+ && matchingVectorsReader.getCompressionMode() == compressionMode+ && matchingVectorsReader.getChunkSize() == chunkSize+ && matchingVectorsReader.getVersion() == VERSION_CURRENT+ && matchingVectorsReader.getPackedIntsVersion() == PackedInts.VERSION_CURRENT+ && BULK_MERGE_ENABLED+ && liveDocs == null+ && !tooDirty(matchingVectorsReader)) {+ // optimized merge, raw byte copy+ // its not worth fine-graining this if there are deletions.++ matchingVectorsReader.checkIntegrity();++ // flush any pending chunks+ if (!pendingDocs.isEmpty()) {+ flush();+ numDirtyChunks++; // incomplete: we had to force this flush+ }++ // iterate over each chunk. we use the vectors index to find chunk boundaries,+ // read the docstart + doccount from the chunk header (we write a new header, since doc+ // numbers will change),+ // and just copy the bytes directly.+ IndexInput rawDocs = matchingVectorsReader.getVectorsStream();+ FieldsIndex index = matchingVectorsReader.getIndexReader();+ rawDocs.seek(index.getStartPointer(0));+ int docID = 0;+ while (docID < maxDoc) {+ // read header+ int base = rawDocs.readVInt();+ if (base != docID) {+ throw new CorruptIndexException(+ "invalid state: base=" + base + ", docID=" + docID, rawDocs);+ }+ int bufferedDocs = rawDocs.readVInt();++ // write a new index entry and new header for this chunk.+ indexWriter.writeIndex(bufferedDocs, vectorsStream.getFilePointer());+ vectorsStream.writeVInt(docCount); // rebase+ vectorsStream.writeVInt(bufferedDocs);+ docID += bufferedDocs;+ docCount += bufferedDocs;+ numDocs += bufferedDocs;++ if (docID > maxDoc) {+ throw new CorruptIndexException(+ "invalid state: base=" + base + ", count=" + bufferedDocs + ", maxDoc=" + maxDoc,+ rawDocs);+ }++ // copy bytes until the next chunk boundary (or end of chunk data).+ // using the stored fields index for this isn't the most efficient, but fast enough+ // and is a source of redundancy for detecting bad things.+ final long end;+ if (docID == maxDoc) {+ end = matchingVectorsReader.getMaxPointer();+ } else {+ end = index.getStartPointer(docID);+ }+ vectorsStream.copyBytes(rawDocs, end - rawDocs.getFilePointer());+ }++ if (rawDocs.getFilePointer() != matchingVectorsReader.getMaxPointer()) {+ throw new CorruptIndexException(+ "invalid state: pos="+ + rawDocs.getFilePointer()+ + ", max="+ + matchingVectorsReader.getMaxPointer(),+ rawDocs);+ }++ // since we bulk merged all chunks, we inherit any dirty ones from this segment.+ numDirtyChunks += matchingVectorsReader.getNumDirtyChunks();+ numDirtyDocs += matchingVectorsReader.getNumDirtyDocs();+ } else {+ // naive merge...+ if (vectorsReader != null) {+ vectorsReader.checkIntegrity();+ }+ for (int i = 0; i < maxDoc; i++) {+ if (liveDocs != null && liveDocs.get(i) == false) {+ continue;+ }+ Fields vectors;+ if (vectorsReader == null) {+ vectors = null;+ } else {+ vectors = vectorsReader.get(i);+ }+ addAllDocVectors(vectors, mergeState);+ ++docCount;+ }+ }+ }+ finish(mergeState.mergeFieldInfos, docCount);+ return docCount;+ }++ /**+ * Returns true if we should recompress this reader, even though we could bulk merge compressed+ * data+ *+ * <p>The last chunk written for a segment is typically incomplete, so without recompressing, in+ * some worst-case situations (e.g. frequent reopen with tiny flushes), over time the compression+ * ratio can degrade. This is a safety switch.+ */+ boolean tooDirty(Lucene50CompressingTermVectorsReader candidate) {+ // more than 1% dirty, or more than hard limit of 1024 dirty chunks+ return candidate.getNumDirtyChunks() > 1024+ || candidate.getNumDirtyDocs() * 100 > candidate.getNumDocs();+ }++ @Override+ public long ramBytesUsed() {+ return positionsBuf.length+ + startOffsetsBuf.length+ + lengthsBuf.length+ + payloadLengthsBuf.length+ + termSuffixes.ramBytesUsed()+ + payloadBytes.ramBytesUsed()+ + lastTerm.bytes.length;+ }++ @Override+ public Collection<Accountable> getChildResources() {+ return List.of(termSuffixes, payloadBytes);+ }+}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50RWCompressingTermVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50RWCompressingTermVectorsFormat.javanew file mode 100644index 00000000000..68ef482bb85--- /dev/null+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50RWCompressingTermVectorsFormat.java@@ -0,0 +1,52 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.backward_codecs.lucene50.compressing;++import java.io.IOException;+import org.apache.lucene.codecs.TermVectorsWriter;+import org.apache.lucene.codecs.compressing.CompressionMode;+import org.apache.lucene.index.SegmentInfo;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.IOContext;++/** RW impersonation of Lucene50CompressingTermVectorsFormat. */+public class Lucene50RWCompressingTermVectorsFormat extends Lucene50CompressingTermVectorsFormat {++ /** Sole constructor. */+ public Lucene50RWCompressingTermVectorsFormat(+ String formatName,+ String segmentSuffix,+ CompressionMode compressionMode,+ int chunkSize,+ int blockSize) {+ super(formatName, segmentSuffix, compressionMode, chunkSize, blockSize);+ }++ @Override+ public final TermVectorsWriter vectorsWriter(+ Directory directory, SegmentInfo segmentInfo, IOContext context) throws IOException {+ return new Lucene50CompressingTermVectorsWriter(+ directory,+ segmentInfo,+ segmentSuffix,+ context,+ formatName,+ compressionMode,+ chunkSize,+ blockSize);+ }+}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene70/Lucene70RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene70/Lucene70RWCodec.javaindex 7b44821765e..5a38dd4f652 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene70/Lucene70RWCodec.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene70/Lucene70RWCodec.java@@ -19,11 +19,13 @@ package org.apache.lucene.backward_codecs.lucene70;import org.apache.lucene.backward_codecs.lucene50.Lucene50RWCompoundFormat;import org.apache.lucene.backward_codecs.lucene50.Lucene50RWPostingsFormat;import org.apache.lucene.backward_codecs.lucene50.Lucene50RWStoredFieldsFormat;+import org.apache.lucene.backward_codecs.lucene50.Lucene50RWTermVectorsFormat;import org.apache.lucene.codecs.CompoundFormat;import org.apache.lucene.codecs.NormsFormat;import org.apache.lucene.codecs.PostingsFormat;import org.apache.lucene.codecs.SegmentInfoFormat;import org.apache.lucene.codecs.StoredFieldsFormat;+import org.apache.lucene.codecs.TermVectorsFormat;import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;/** RW impersonation of {@link Lucene70Codec}. */@@ -65,4 +67,9 @@ public final class Lucene70RWCodec extends Lucene70Codec {public CompoundFormat compoundFormat() {return new Lucene50RWCompoundFormat();}++ @Override+ public TermVectorsFormat termVectorsFormat() {+ return new Lucene50RWTermVectorsFormat();+ }}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84RWCodec.javaindex 3b70b2c0edd..5d578d065ed 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84RWCodec.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene84/Lucene84RWCodec.java@@ -18,6 +18,7 @@ package org.apache.lucene.backward_codecs.lucene84;import org.apache.lucene.backward_codecs.lucene50.Lucene50RWCompoundFormat;import org.apache.lucene.backward_codecs.lucene50.Lucene50RWStoredFieldsFormat;+import org.apache.lucene.backward_codecs.lucene50.Lucene50RWTermVectorsFormat;import org.apache.lucene.backward_codecs.lucene60.Lucene60RWPointsFormat;import org.apache.lucene.backward_codecs.lucene70.Lucene70RWSegmentInfoFormat;import org.apache.lucene.backward_codecs.lucene80.Lucene80RWNormsFormat;@@ -27,6 +28,7 @@ import org.apache.lucene.codecs.PointsFormat;import org.apache.lucene.codecs.PostingsFormat;import org.apache.lucene.codecs.SegmentInfoFormat;import org.apache.lucene.codecs.StoredFieldsFormat;+import org.apache.lucene.codecs.TermVectorsFormat;import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;/** RW impersonation of {@link Lucene84Codec}. */@@ -70,4 +72,9 @@ public class Lucene84RWCodec extends Lucene84Codec {public final CompoundFormat compoundFormat() {return new Lucene50RWCompoundFormat();}++ @Override+ public TermVectorsFormat termVectorsFormat() {+ return new Lucene50RWTermVectorsFormat();+ }}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86RWCodec.javaindex 97d56333187..6a544fd45c6 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86RWCodec.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene86/Lucene86RWCodec.java@@ -18,6 +18,7 @@ package org.apache.lucene.backward_codecs.lucene86;import org.apache.lucene.backward_codecs.lucene50.Lucene50RWCompoundFormat;import org.apache.lucene.backward_codecs.lucene50.Lucene50RWStoredFieldsFormat;+import org.apache.lucene.backward_codecs.lucene50.Lucene50RWTermVectorsFormat;import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat;import org.apache.lucene.backward_codecs.lucene80.Lucene80RWNormsFormat;import org.apache.lucene.backward_codecs.lucene84.Lucene84RWPostingsFormat;@@ -25,6 +26,7 @@ import org.apache.lucene.codecs.CompoundFormat;import org.apache.lucene.codecs.NormsFormat;import org.apache.lucene.codecs.PostingsFormat;import org.apache.lucene.codecs.StoredFieldsFormat;+import org.apache.lucene.codecs.TermVectorsFormat;import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;/** RW impersonation of {@link Lucene86Codec}. */@@ -69,4 +71,9 @@ public class Lucene86RWCodec extends Lucene86Codec {public final CompoundFormat compoundFormat() {return new Lucene50RWCompoundFormat();}++ @Override+ public TermVectorsFormat termVectorsFormat() {+ return new Lucene50RWTermVectorsFormat();+ }}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/Lucene87RWCodec.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/Lucene87RWCodec.javaindex 8fac2d6076b..a5af4089e1f 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/Lucene87RWCodec.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/Lucene87RWCodec.java@@ -17,11 +17,13 @@package org.apache.lucene.backward_codecs.lucene87;import org.apache.lucene.backward_codecs.lucene50.Lucene50RWCompoundFormat;+import org.apache.lucene.backward_codecs.lucene50.Lucene50RWTermVectorsFormat;import org.apache.lucene.backward_codecs.lucene80.Lucene80RWNormsFormat;import org.apache.lucene.backward_codecs.lucene84.Lucene84RWPostingsFormat;import org.apache.lucene.codecs.CompoundFormat;import org.apache.lucene.codecs.NormsFormat;import org.apache.lucene.codecs.PostingsFormat;+import org.apache.lucene.codecs.TermVectorsFormat;import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;/** RW impersonation of {@link Lucene87Codec}. */@@ -50,4 +52,9 @@ public class Lucene87RWCodec extends Lucene87Codec {public PostingsFormat postingsFormat() {return postingsFormat;}++ @Override+ public TermVectorsFormat termVectorsFormat() {+ return new Lucene50RWTermVectorsFormat();+ }}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/Lucene90CompressingTermVectorsFormat.javasimilarity index 91%rename from lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsFormat.javarename to lucene/core/src/java/org/apache/lucene/codecs/compressing/Lucene90CompressingTermVectorsFormat.javaindex 15016b81843..13e070848d0 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsFormat.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/Lucene90CompressingTermVectorsFormat.java@@ -33,7 +33,7 @@ import org.apache.lucene.store.IOContext;** @lucene.experimental*/-public class CompressingTermVectorsFormat extends TermVectorsFormat {+public class Lucene90CompressingTermVectorsFormat extends TermVectorsFormat {private final String formatName;private final String segmentSuffix;@@ -42,7 +42,7 @@ public class CompressingTermVectorsFormat extends TermVectorsFormat {private final int blockSize;/**- * Create a new {@link CompressingTermVectorsFormat}.+ * Create a new {@link Lucene90CompressingTermVectorsFormat}.** <p><code>formatName</code> is the name of the format. This name will be used in the file* formats to perform {@link CodecUtil#checkIndexHeader codec header checks}.@@ -50,7 +50,8 @@ public class CompressingTermVectorsFormat extends TermVectorsFormat {* <p>The <code>compressionMode</code> parameter allows you to choose between compression* algorithms that have various compression and decompression speeds so that you can pick the one* that best fits your indexing and searching throughput. You should never instantiate two {@link- * CompressingTermVectorsFormat}s that have the same name but different {@link CompressionMode}s.+ * Lucene90CompressingTermVectorsFormat}s that have the same name but different {@link+ * CompressionMode}s.** <p><code>chunkSize</code> is the minimum byte size of a chunk of documents. Higher values of* <code>chunkSize</code> should improve the compression ratio but will require more memory at@@ -64,7 +65,7 @@ public class CompressingTermVectorsFormat extends TermVectorsFormat {* @param blockSize the number of chunks to store in an index block.* @see CompressionMode*/- public CompressingTermVectorsFormat(+ public Lucene90CompressingTermVectorsFormat(String formatName,String segmentSuffix,CompressionMode compressionMode,@@ -87,14 +88,14 @@ public class CompressingTermVectorsFormat extends TermVectorsFormat {public final TermVectorsReader vectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context)throws IOException {- return new CompressingTermVectorsReader(+ return new Lucene90CompressingTermVectorsReader(directory, segmentInfo, segmentSuffix, fieldInfos, context, formatName, compressionMode);}@Overridepublic final TermVectorsWriter vectorsWriter(Directory directory, SegmentInfo segmentInfo, IOContext context) throws IOException {- return new CompressingTermVectorsWriter(+ return new Lucene90CompressingTermVectorsWriter(directory,segmentInfo,segmentSuffix,diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/Lucene90CompressingTermVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/Lucene90CompressingTermVectorsReader.javanew file mode 100644index 00000000000..c31a37f371d--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/Lucene90CompressingTermVectorsReader.java@@ -0,0 +1,1299 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.compressing;++import static org.apache.lucene.codecs.compressing.Lucene90CompressingTermVectorsWriter.FLAGS_BITS;+import static org.apache.lucene.codecs.compressing.Lucene90CompressingTermVectorsWriter.META_VERSION_START;+import static org.apache.lucene.codecs.compressing.Lucene90CompressingTermVectorsWriter.OFFSETS;+import static org.apache.lucene.codecs.compressing.Lucene90CompressingTermVectorsWriter.PACKED_BLOCK_SIZE;+import static org.apache.lucene.codecs.compressing.Lucene90CompressingTermVectorsWriter.PAYLOADS;+import static org.apache.lucene.codecs.compressing.Lucene90CompressingTermVectorsWriter.POSITIONS;+import static org.apache.lucene.codecs.compressing.Lucene90CompressingTermVectorsWriter.VECTORS_EXTENSION;+import static org.apache.lucene.codecs.compressing.Lucene90CompressingTermVectorsWriter.VECTORS_INDEX_CODEC_NAME;+import static org.apache.lucene.codecs.compressing.Lucene90CompressingTermVectorsWriter.VECTORS_INDEX_EXTENSION;+import static org.apache.lucene.codecs.compressing.Lucene90CompressingTermVectorsWriter.VECTORS_META_EXTENSION;+import static org.apache.lucene.codecs.compressing.Lucene90CompressingTermVectorsWriter.VERSION_CURRENT;+import static org.apache.lucene.codecs.compressing.Lucene90CompressingTermVectorsWriter.VERSION_START;++import java.io.Closeable;+import java.io.IOException;+import java.util.Collection;+import java.util.Collections;+import java.util.Iterator;+import java.util.NoSuchElementException;+import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.codecs.TermVectorsReader;+import org.apache.lucene.index.BaseTermsEnum;+import org.apache.lucene.index.CorruptIndexException;+import org.apache.lucene.index.FieldInfo;+import org.apache.lucene.index.FieldInfos;+import org.apache.lucene.index.Fields;+import org.apache.lucene.index.ImpactsEnum;+import org.apache.lucene.index.IndexFileNames;+import org.apache.lucene.index.PostingsEnum;+import org.apache.lucene.index.SegmentInfo;+import org.apache.lucene.index.SlowImpactsEnum;+import org.apache.lucene.index.Terms;+import org.apache.lucene.index.TermsEnum;+import org.apache.lucene.store.AlreadyClosedException;+import org.apache.lucene.store.ByteArrayDataInput;+import org.apache.lucene.store.ChecksumIndexInput;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.util.Accountable;+import org.apache.lucene.util.Accountables;+import org.apache.lucene.util.ArrayUtil;+import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.IOUtils;+import org.apache.lucene.util.LongsRef;+import org.apache.lucene.util.packed.BlockPackedReaderIterator;+import org.apache.lucene.util.packed.PackedInts;++/**+ * {@link TermVectorsReader} for {@link Lucene90CompressingTermVectorsFormat}.+ *+ * @lucene.experimental+ */+public final class Lucene90CompressingTermVectorsReader extends TermVectorsReader+ implements Closeable {++ private final FieldInfos fieldInfos;+ final FieldsIndex indexReader;+ final IndexInput vectorsStream;+ private final int version;+ private final int packedIntsVersion;+ private final CompressionMode compressionMode;+ private final Decompressor decompressor;+ private final int chunkSize;+ private final int numDocs;+ private boolean closed;+ private final BlockPackedReaderIterator reader;+ private final long numDirtyChunks; // number of incomplete compressed blocks written+ private final long numDirtyDocs; // cumulative number of missing docs in incomplete chunks+ private final long maxPointer; // end of the data section++ // used by clone+ private Lucene90CompressingTermVectorsReader(Lucene90CompressingTermVectorsReader reader) {+ this.fieldInfos = reader.fieldInfos;+ this.vectorsStream = reader.vectorsStream.clone();+ this.indexReader = reader.indexReader.clone();+ this.packedIntsVersion = reader.packedIntsVersion;+ this.compressionMode = reader.compressionMode;+ this.decompressor = reader.decompressor.clone();+ this.chunkSize = reader.chunkSize;+ this.numDocs = reader.numDocs;+ this.reader =+ new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, PACKED_BLOCK_SIZE, 0);+ this.version = reader.version;+ this.numDirtyChunks = reader.numDirtyChunks;+ this.numDirtyDocs = reader.numDirtyDocs;+ this.maxPointer = reader.maxPointer;+ this.closed = false;+ }++ /** Sole constructor. */+ public Lucene90CompressingTermVectorsReader(+ Directory d,+ SegmentInfo si,+ String segmentSuffix,+ FieldInfos fn,+ IOContext context,+ String formatName,+ CompressionMode compressionMode)+ throws IOException {+ this.compressionMode = compressionMode;+ final String segment = si.name;+ boolean success = false;+ fieldInfos = fn;+ numDocs = si.maxDoc();++ ChecksumIndexInput metaIn = null;+ try {+ // Open the data file+ final String vectorsStreamFN =+ IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION);+ vectorsStream = d.openInput(vectorsStreamFN, context);+ version =+ CodecUtil.checkIndexHeader(+ vectorsStream, formatName, VERSION_START, VERSION_CURRENT, si.getId(), segmentSuffix);+ assert CodecUtil.indexHeaderLength(formatName, segmentSuffix)+ == vectorsStream.getFilePointer();++ final String metaStreamFN =+ IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_META_EXTENSION);+ metaIn = d.openChecksumInput(metaStreamFN, IOContext.READONCE);+ CodecUtil.checkIndexHeader(+ metaIn,+ VECTORS_INDEX_CODEC_NAME + "Meta",+ META_VERSION_START,+ version,+ si.getId(),+ segmentSuffix);++ packedIntsVersion = metaIn.readVInt();+ chunkSize = metaIn.readVInt();++ // NOTE: data file is too costly to verify checksum against all the bytes on open,+ // but for now we at least verify proper structure of the checksum footer: which looks+ // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption+ // such as file truncation.+ CodecUtil.retrieveChecksum(vectorsStream);++ FieldsIndexReader fieldsIndexReader =+ new FieldsIndexReader(+ d,+ si.name,+ segmentSuffix,+ VECTORS_INDEX_EXTENSION,+ VECTORS_INDEX_CODEC_NAME,+ si.getId(),+ metaIn);++ this.indexReader = fieldsIndexReader;+ this.maxPointer = fieldsIndexReader.getMaxPointer();++ numDirtyChunks = metaIn.readVLong();+ numDirtyDocs = metaIn.readVLong();++ decompressor = compressionMode.newDecompressor();+ this.reader =+ new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, PACKED_BLOCK_SIZE, 0);++ if (metaIn != null) {+ CodecUtil.checkFooter(metaIn, null);+ metaIn.close();+ }++ success = true;+ } catch (Throwable t) {+ if (metaIn != null) {+ CodecUtil.checkFooter(metaIn, t);+ throw new AssertionError("unreachable");+ } else {+ throw t;+ }+ } finally {+ if (!success) {+ IOUtils.closeWhileHandlingException(this, metaIn);+ }+ }+ }++ CompressionMode getCompressionMode() {+ return compressionMode;+ }++ int getChunkSize() {+ return chunkSize;+ }++ int getPackedIntsVersion() {+ return packedIntsVersion;+ }++ int getVersion() {+ return version;+ }++ FieldsIndex getIndexReader() {+ return indexReader;+ }++ IndexInput getVectorsStream() {+ return vectorsStream;+ }++ long getMaxPointer() {+ return maxPointer;+ }++ long getNumDirtyDocs() {+ if (version != VERSION_CURRENT) {+ throw new IllegalStateException(+ "getNumDirtyDocs should only ever get called when the reader is on the current version");+ }+ assert numDirtyDocs >= 0;+ return numDirtyDocs;+ }++ long getNumDirtyChunks() {+ if (version != VERSION_CURRENT) {+ throw new IllegalStateException(+ "getNumDirtyChunks should only ever get called when the reader is on the current version");+ }+ assert numDirtyChunks >= 0;+ return numDirtyChunks;+ }++ int getNumDocs() {+ return numDocs;+ }++ /** @throws AlreadyClosedException if this TermVectorsReader is closed */+ private void ensureOpen() throws AlreadyClosedException {+ if (closed) {+ throw new AlreadyClosedException("this FieldsReader is closed");+ }+ }++ @Override+ public void close() throws IOException {+ if (!closed) {+ IOUtils.close(indexReader, vectorsStream);+ closed = true;+ }+ }++ @Override+ public TermVectorsReader clone() {+ return new Lucene90CompressingTermVectorsReader(this);+ }++ @Override+ public Fields get(int doc) throws IOException {+ ensureOpen();++ // seek to the right place+ {+ final long startPointer = indexReader.getStartPointer(doc);+ vectorsStream.seek(startPointer);+ }++ // decode+ // - docBase: first doc ID of the chunk+ // - chunkDocs: number of docs of the chunk+ final int docBase = vectorsStream.readVInt();+ final int chunkDocs = vectorsStream.readVInt();+ if (doc < docBase || doc >= docBase + chunkDocs || docBase + chunkDocs > numDocs) {+ throw new CorruptIndexException(+ "docBase=" + docBase + ",chunkDocs=" + chunkDocs + ",doc=" + doc, vectorsStream);+ }++ final int skip; // number of fields to skip+ final int numFields; // number of fields of the document we're looking for+ final int totalFields; // total number of fields of the chunk (sum for all docs)+ if (chunkDocs == 1) {+ skip = 0;+ numFields = totalFields = vectorsStream.readVInt();+ } else {+ reader.reset(vectorsStream, chunkDocs);+ int sum = 0;+ for (int i = docBase; i < doc; ++i) {+ sum += reader.next();+ }+ skip = sum;+ numFields = (int) reader.next();+ sum += numFields;+ for (int i = doc + 1; i < docBase + chunkDocs; ++i) {+ sum += reader.next();+ }+ totalFields = sum;+ }++ if (numFields == 0) {+ // no vectors+ return null;+ }++ // read field numbers that have term vectors+ final int[] fieldNums;+ {+ final int token = vectorsStream.readByte() & 0xFF;+ assert token != 0; // means no term vectors, cannot happen since we checked for numFields == 0+ final int bitsPerFieldNum = token & 0x1F;+ int totalDistinctFields = token >>> 5;+ if (totalDistinctFields == 0x07) {+ totalDistinctFields += vectorsStream.readVInt();+ }+ ++totalDistinctFields;+ final PackedInts.ReaderIterator it =+ PackedInts.getReaderIteratorNoHeader(+ vectorsStream,+ PackedInts.Format.PACKED,+ packedIntsVersion,+ totalDistinctFields,+ bitsPerFieldNum,+ 1);+ fieldNums = new int[totalDistinctFields];+ for (int i = 0; i < totalDistinctFields; ++i) {+ fieldNums[i] = (int) it.next();+ }+ }++ // read field numbers and flags+ final int[] fieldNumOffs = new int[numFields];+ final PackedInts.Reader flags;+ {+ final int bitsPerOff = PackedInts.bitsRequired(fieldNums.length - 1);+ final PackedInts.Reader allFieldNumOffs =+ PackedInts.getReaderNoHeader(+ vectorsStream, PackedInts.Format.PACKED, packedIntsVersion, totalFields, bitsPerOff);+ switch (vectorsStream.readVInt()) {+ case 0:+ final PackedInts.Reader fieldFlags =+ PackedInts.getReaderNoHeader(+ vectorsStream,+ PackedInts.Format.PACKED,+ packedIntsVersion,+ fieldNums.length,+ FLAGS_BITS);+ PackedInts.Mutable f = PackedInts.getMutable(totalFields, FLAGS_BITS, PackedInts.COMPACT);+ for (int i = 0; i < totalFields; ++i) {+ final int fieldNumOff = (int) allFieldNumOffs.get(i);+ assert fieldNumOff >= 0 && fieldNumOff < fieldNums.length;+ final int fgs = (int) fieldFlags.get(fieldNumOff);+ f.set(i, fgs);+ }+ flags = f;+ break;+ case 1:+ flags =+ PackedInts.getReaderNoHeader(+ vectorsStream,+ PackedInts.Format.PACKED,+ packedIntsVersion,+ totalFields,+ FLAGS_BITS);+ break;+ default:+ throw new AssertionError();+ }+ for (int i = 0; i < numFields; ++i) {+ fieldNumOffs[i] = (int) allFieldNumOffs.get(skip + i);+ }+ }++ // number of terms per field for all fields+ final PackedInts.Reader numTerms;+ final int totalTerms;+ {+ final int bitsRequired = vectorsStream.readVInt();+ numTerms =+ PackedInts.getReaderNoHeader(+ vectorsStream,+ PackedInts.Format.PACKED,+ packedIntsVersion,+ totalFields,+ bitsRequired);+ int sum = 0;+ for (int i = 0; i < totalFields; ++i) {+ sum += numTerms.get(i);+ }+ totalTerms = sum;+ }++ // term lengths+ int docOff = 0, docLen = 0, totalLen;+ final int[] fieldLengths = new int[numFields];+ final int[][] prefixLengths = new int[numFields][];+ final int[][] suffixLengths = new int[numFields][];+ {+ reader.reset(vectorsStream, totalTerms);+ // skip+ int toSkip = 0;+ for (int i = 0; i < skip; ++i) {+ toSkip += numTerms.get(i);+ }+ reader.skip(toSkip);+ // read prefix lengths+ for (int i = 0; i < numFields; ++i) {+ final int termCount = (int) numTerms.get(skip + i);+ final int[] fieldPrefixLengths = new int[termCount];+ prefixLengths[i] = fieldPrefixLengths;+ for (int j = 0; j < termCount; ) {+ final LongsRef next = reader.next(termCount - j);+ for (int k = 0; k < next.length; ++k) {+ fieldPrefixLengths[j++] = (int) next.longs[next.offset + k];+ }+ }+ }+ reader.skip(totalTerms - reader.ord());++ reader.reset(vectorsStream, totalTerms);+ // skip+ toSkip = 0;+ for (int i = 0; i < skip; ++i) {+ for (int j = 0; j < numTerms.get(i); ++j) {+ docOff += reader.next();+ }+ }+ for (int i = 0; i < numFields; ++i) {+ final int termCount = (int) numTerms.get(skip + i);+ final int[] fieldSuffixLengths = new int[termCount];+ suffixLengths[i] = fieldSuffixLengths;+ for (int j = 0; j < termCount; ) {+ final LongsRef next = reader.next(termCount - j);+ for (int k = 0; k < next.length; ++k) {+ fieldSuffixLengths[j++] = (int) next.longs[next.offset + k];+ }+ }+ fieldLengths[i] = sum(suffixLengths[i]);+ docLen += fieldLengths[i];+ }+ totalLen = docOff + docLen;+ for (int i = skip + numFields; i < totalFields; ++i) {+ for (int j = 0; j < numTerms.get(i); ++j) {+ totalLen += reader.next();+ }+ }+ }++ // term freqs+ final int[] termFreqs = new int[totalTerms];+ {+ reader.reset(vectorsStream, totalTerms);+ for (int i = 0; i < totalTerms; ) {+ final LongsRef next = reader.next(totalTerms - i);+ for (int k = 0; k < next.length; ++k) {+ termFreqs[i++] = 1 + (int) next.longs[next.offset + k];+ }+ }+ }++ // total number of positions, offsets and payloads+ int totalPositions = 0, totalOffsets = 0, totalPayloads = 0;+ for (int i = 0, termIndex = 0; i < totalFields; ++i) {+ final int f = (int) flags.get(i);+ final int termCount = (int) numTerms.get(i);+ for (int j = 0; j < termCount; ++j) {+ final int freq = termFreqs[termIndex++];+ if ((f & POSITIONS) != 0) {+ totalPositions += freq;+ }+ if ((f & OFFSETS) != 0) {+ totalOffsets += freq;+ }+ if ((f & PAYLOADS) != 0) {+ totalPayloads += freq;+ }+ }+ assert i != totalFields - 1 || termIndex == totalTerms : termIndex + " " + totalTerms;+ }++ final int[][] positionIndex = positionIndex(skip, numFields, numTerms, termFreqs);+ final int[][] positions, startOffsets, lengths;+ if (totalPositions > 0) {+ positions =+ readPositions(+ skip,+ numFields,+ flags,+ numTerms,+ termFreqs,+ POSITIONS,+ totalPositions,+ positionIndex);+ } else {+ positions = new int[numFields][];+ }++ if (totalOffsets > 0) {+ // average number of chars per term+ final float[] charsPerTerm = new float[fieldNums.length];+ for (int i = 0; i < charsPerTerm.length; ++i) {+ charsPerTerm[i] = Float.intBitsToFloat(vectorsStream.readInt());+ }+ startOffsets =+ readPositions(+ skip, numFields, flags, numTerms, termFreqs, OFFSETS, totalOffsets, positionIndex);+ lengths =+ readPositions(+ skip, numFields, flags, numTerms, termFreqs, OFFSETS, totalOffsets, positionIndex);++ for (int i = 0; i < numFields; ++i) {+ final int[] fStartOffsets = startOffsets[i];+ final int[] fPositions = positions[i];+ // patch offsets from positions+ if (fStartOffsets != null && fPositions != null) {+ final float fieldCharsPerTerm = charsPerTerm[fieldNumOffs[i]];+ for (int j = 0; j < startOffsets[i].length; ++j) {+ fStartOffsets[j] += (int) (fieldCharsPerTerm * fPositions[j]);+ }+ }+ if (fStartOffsets != null) {+ final int[] fPrefixLengths = prefixLengths[i];+ final int[] fSuffixLengths = suffixLengths[i];+ final int[] fLengths = lengths[i];+ for (int j = 0, end = (int) numTerms.get(skip + i); j < end; ++j) {+ // delta-decode start offsets and patch lengths using term lengths+ final int termLength = fPrefixLengths[j] + fSuffixLengths[j];+ lengths[i][positionIndex[i][j]] += termLength;+ for (int k = positionIndex[i][j] + 1; k < positionIndex[i][j + 1]; ++k) {+ fStartOffsets[k] += fStartOffsets[k - 1];+ fLengths[k] += termLength;+ }+ }+ }+ }+ } else {+ startOffsets = lengths = new int[numFields][];+ }+ if (totalPositions > 0) {+ // delta-decode positions+ for (int i = 0; i < numFields; ++i) {+ final int[] fPositions = positions[i];+ final int[] fpositionIndex = positionIndex[i];+ if (fPositions != null) {+ for (int j = 0, end = (int) numTerms.get(skip + i); j < end; ++j) {+ // delta-decode start offsets+ for (int k = fpositionIndex[j] + 1; k < fpositionIndex[j + 1]; ++k) {+ fPositions[k] += fPositions[k - 1];+ }+ }+ }+ }+ }++ // payload lengths+ final int[][] payloadIndex = new int[numFields][];+ int totalPayloadLength = 0;+ int payloadOff = 0;+ int payloadLen = 0;+ if (totalPayloads > 0) {+ reader.reset(vectorsStream, totalPayloads);+ // skip+ int termIndex = 0;+ for (int i = 0; i < skip; ++i) {+ final int f = (int) flags.get(i);+ final int termCount = (int) numTerms.get(i);+ if ((f & PAYLOADS) != 0) {+ for (int j = 0; j < termCount; ++j) {+ final int freq = termFreqs[termIndex + j];+ for (int k = 0; k < freq; ++k) {+ final int l = (int) reader.next();+ payloadOff += l;+ }+ }+ }+ termIndex += termCount;+ }+ totalPayloadLength = payloadOff;+ // read doc payload lengths+ for (int i = 0; i < numFields; ++i) {+ final int f = (int) flags.get(skip + i);+ final int termCount = (int) numTerms.get(skip + i);+ if ((f & PAYLOADS) != 0) {+ final int totalFreq = positionIndex[i][termCount];+ payloadIndex[i] = new int[totalFreq + 1];+ int posIdx = 0;+ payloadIndex[i][posIdx] = payloadLen;+ for (int j = 0; j < termCount; ++j) {+ final int freq = termFreqs[termIndex + j];+ for (int k = 0; k < freq; ++k) {+ final int payloadLength = (int) reader.next();+ payloadLen += payloadLength;+ payloadIndex[i][posIdx + 1] = payloadLen;+ ++posIdx;+ }+ }+ assert posIdx == totalFreq;+ }+ termIndex += termCount;+ }+ totalPayloadLength += payloadLen;+ for (int i = skip + numFields; i < totalFields; ++i) {+ final int f = (int) flags.get(i);+ final int termCount = (int) numTerms.get(i);+ if ((f & PAYLOADS) != 0) {+ for (int j = 0; j < termCount; ++j) {+ final int freq = termFreqs[termIndex + j];+ for (int k = 0; k < freq; ++k) {+ totalPayloadLength += reader.next();+ }+ }+ }+ termIndex += termCount;+ }+ assert termIndex == totalTerms : termIndex + " " + totalTerms;+ }++ // decompress data+ final BytesRef suffixBytes = new BytesRef();+ decompressor.decompress(+ vectorsStream,+ totalLen + totalPayloadLength,+ docOff + payloadOff,+ docLen + payloadLen,+ suffixBytes);+ suffixBytes.length = docLen;+ final BytesRef payloadBytes =+ new BytesRef(suffixBytes.bytes, suffixBytes.offset + docLen, payloadLen);++ final int[] fieldFlags = new int[numFields];+ for (int i = 0; i < numFields; ++i) {+ fieldFlags[i] = (int) flags.get(skip + i);+ }++ final int[] fieldNumTerms = new int[numFields];+ for (int i = 0; i < numFields; ++i) {+ fieldNumTerms[i] = (int) numTerms.get(skip + i);+ }++ final int[][] fieldTermFreqs = new int[numFields][];+ {+ int termIdx = 0;+ for (int i = 0; i < skip; ++i) {+ termIdx += numTerms.get(i);+ }+ for (int i = 0; i < numFields; ++i) {+ final int termCount = (int) numTerms.get(skip + i);+ fieldTermFreqs[i] = new int[termCount];+ for (int j = 0; j < termCount; ++j) {+ fieldTermFreqs[i][j] = termFreqs[termIdx++];+ }+ }+ }++ assert sum(fieldLengths) == docLen : sum(fieldLengths) + " != " + docLen;++ return new TVFields(+ fieldNums,+ fieldFlags,+ fieldNumOffs,+ fieldNumTerms,+ fieldLengths,+ prefixLengths,+ suffixLengths,+ fieldTermFreqs,+ positionIndex,+ positions,+ startOffsets,+ lengths,+ payloadBytes,+ payloadIndex,+ suffixBytes);+ }++ // field -> term index -> position index+ private int[][] positionIndex(+ int skip, int numFields, PackedInts.Reader numTerms, int[] termFreqs) {+ final int[][] positionIndex = new int[numFields][];+ int termIndex = 0;+ for (int i = 0; i < skip; ++i) {+ final int termCount = (int) numTerms.get(i);+ termIndex += termCount;+ }+ for (int i = 0; i < numFields; ++i) {+ final int termCount = (int) numTerms.get(skip + i);+ positionIndex[i] = new int[termCount + 1];+ for (int j = 0; j < termCount; ++j) {+ final int freq = termFreqs[termIndex + j];+ positionIndex[i][j + 1] = positionIndex[i][j] + freq;+ }+ termIndex += termCount;+ }+ return positionIndex;+ }++ private int[][] readPositions(+ int skip,+ int numFields,+ PackedInts.Reader flags,+ PackedInts.Reader numTerms,+ int[] termFreqs,+ int flag,+ final int totalPositions,+ int[][] positionIndex)+ throws IOException {+ final int[][] positions = new int[numFields][];+ reader.reset(vectorsStream, totalPositions);+ // skip+ int toSkip = 0;+ int termIndex = 0;+ for (int i = 0; i < skip; ++i) {+ final int f = (int) flags.get(i);+ final int termCount = (int) numTerms.get(i);+ if ((f & flag) != 0) {+ for (int j = 0; j < termCount; ++j) {+ final int freq = termFreqs[termIndex + j];+ toSkip += freq;+ }+ }+ termIndex += termCount;+ }+ reader.skip(toSkip);+ // read doc positions+ for (int i = 0; i < numFields; ++i) {+ final int f = (int) flags.get(skip + i);+ final int termCount = (int) numTerms.get(skip + i);+ if ((f & flag) != 0) {+ final int totalFreq = positionIndex[i][termCount];+ final int[] fieldPositions = new int[totalFreq];+ positions[i] = fieldPositions;+ for (int j = 0; j < totalFreq; ) {+ final LongsRef nextPositions = reader.next(totalFreq - j);+ for (int k = 0; k < nextPositions.length; ++k) {+ fieldPositions[j++] = (int) nextPositions.longs[nextPositions.offset + k];+ }+ }+ }+ termIndex += termCount;+ }+ reader.skip(totalPositions - reader.ord());+ return positions;+ }++ private class TVFields extends Fields {++ private final int[] fieldNums, fieldFlags, fieldNumOffs, numTerms, fieldLengths;+ private final int[][] prefixLengths,+ suffixLengths,+ termFreqs,+ positionIndex,+ positions,+ startOffsets,+ lengths,+ payloadIndex;+ private final BytesRef suffixBytes, payloadBytes;++ public TVFields(+ int[] fieldNums,+ int[] fieldFlags,+ int[] fieldNumOffs,+ int[] numTerms,+ int[] fieldLengths,+ int[][] prefixLengths,+ int[][] suffixLengths,+ int[][] termFreqs,+ int[][] positionIndex,+ int[][] positions,+ int[][] startOffsets,+ int[][] lengths,+ BytesRef payloadBytes,+ int[][] payloadIndex,+ BytesRef suffixBytes) {+ this.fieldNums = fieldNums;+ this.fieldFlags = fieldFlags;+ this.fieldNumOffs = fieldNumOffs;+ this.numTerms = numTerms;+ this.fieldLengths = fieldLengths;+ this.prefixLengths = prefixLengths;+ this.suffixLengths = suffixLengths;+ this.termFreqs = termFreqs;+ this.positionIndex = positionIndex;+ this.positions = positions;+ this.startOffsets = startOffsets;+ this.lengths = lengths;+ this.payloadBytes = payloadBytes;+ this.payloadIndex = payloadIndex;+ this.suffixBytes = suffixBytes;+ }++ @Override+ public Iterator<String> iterator() {+ return new Iterator<String>() {+ int i = 0;++ @Override+ public boolean hasNext() {+ return i < fieldNumOffs.length;+ }++ @Override+ public String next() {+ if (!hasNext()) {+ throw new NoSuchElementException();+ }+ final int fieldNum = fieldNums[fieldNumOffs[i++]];+ return fieldInfos.fieldInfo(fieldNum).name;+ }++ @Override+ public void remove() {+ throw new UnsupportedOperationException();+ }+ };+ }++ @Override+ public Terms terms(String field) throws IOException {+ final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);+ if (fieldInfo == null) {+ return null;+ }+ int idx = -1;+ for (int i = 0; i < fieldNumOffs.length; ++i) {+ if (fieldNums[fieldNumOffs[i]] == fieldInfo.number) {+ idx = i;+ break;+ }+ }++ if (idx == -1 || numTerms[idx] == 0) {+ // no term+ return null;+ }+ int fieldOff = 0, fieldLen = -1;+ for (int i = 0; i < fieldNumOffs.length; ++i) {+ if (i < idx) {+ fieldOff += fieldLengths[i];+ } else {+ fieldLen = fieldLengths[i];+ break;+ }+ }+ assert fieldLen >= 0;+ return new TVTerms(+ numTerms[idx],+ fieldFlags[idx],+ prefixLengths[idx],+ suffixLengths[idx],+ termFreqs[idx],+ positionIndex[idx],+ positions[idx],+ startOffsets[idx],+ lengths[idx],+ payloadIndex[idx],+ payloadBytes,+ new BytesRef(suffixBytes.bytes, suffixBytes.offset + fieldOff, fieldLen));+ }++ @Override+ public int size() {+ return fieldNumOffs.length;+ }+ }++ private static class TVTerms extends Terms {++ private final int numTerms, flags;+ private final long totalTermFreq;+ private final int[] prefixLengths,+ suffixLengths,+ termFreqs,+ positionIndex,+ positions,+ startOffsets,+ lengths,+ payloadIndex;+ private final BytesRef termBytes, payloadBytes;++ TVTerms(+ int numTerms,+ int flags,+ int[] prefixLengths,+ int[] suffixLengths,+ int[] termFreqs,+ int[] positionIndex,+ int[] positions,+ int[] startOffsets,+ int[] lengths,+ int[] payloadIndex,+ BytesRef payloadBytes,+ BytesRef termBytes) {+ this.numTerms = numTerms;+ this.flags = flags;+ this.prefixLengths = prefixLengths;+ this.suffixLengths = suffixLengths;+ this.termFreqs = termFreqs;+ this.positionIndex = positionIndex;+ this.positions = positions;+ this.startOffsets = startOffsets;+ this.lengths = lengths;+ this.payloadIndex = payloadIndex;+ this.payloadBytes = payloadBytes;+ this.termBytes = termBytes;+ long ttf = 0;+ for (int tf : termFreqs) {+ ttf += tf;+ }+ this.totalTermFreq = ttf;+ }++ @Override+ public TermsEnum iterator() throws IOException {+ TVTermsEnum termsEnum = new TVTermsEnum();+ termsEnum.reset(+ numTerms,+ flags,+ prefixLengths,+ suffixLengths,+ termFreqs,+ positionIndex,+ positions,+ startOffsets,+ lengths,+ payloadIndex,+ payloadBytes,+ new ByteArrayDataInput(termBytes.bytes, termBytes.offset, termBytes.length));+ return termsEnum;+ }++ @Override+ public long size() throws IOException {+ return numTerms;+ }++ @Override+ public long getSumTotalTermFreq() throws IOException {+ return totalTermFreq;+ }++ @Override+ public long getSumDocFreq() throws IOException {+ return numTerms;+ }++ @Override+ public int getDocCount() throws IOException {+ return 1;+ }++ @Override+ public boolean hasFreqs() {+ return true;+ }++ @Override+ public boolean hasOffsets() {+ return (flags & OFFSETS) != 0;+ }++ @Override+ public boolean hasPositions() {+ return (flags & POSITIONS) != 0;+ }++ @Override+ public boolean hasPayloads() {+ return (flags & PAYLOADS) != 0;+ }+ }++ private static class TVTermsEnum extends BaseTermsEnum {++ private int numTerms, startPos, ord;+ private int[] prefixLengths,+ suffixLengths,+ termFreqs,+ positionIndex,+ positions,+ startOffsets,+ lengths,+ payloadIndex;+ private ByteArrayDataInput in;+ private BytesRef payloads;+ private final BytesRef term;++ private TVTermsEnum() {+ term = new BytesRef(16);+ }++ void reset(+ int numTerms,+ int flags,+ int[] prefixLengths,+ int[] suffixLengths,+ int[] termFreqs,+ int[] positionIndex,+ int[] positions,+ int[] startOffsets,+ int[] lengths,+ int[] payloadIndex,+ BytesRef payloads,+ ByteArrayDataInput in) {+ this.numTerms = numTerms;+ this.prefixLengths = prefixLengths;+ this.suffixLengths = suffixLengths;+ this.termFreqs = termFreqs;+ this.positionIndex = positionIndex;+ this.positions = positions;+ this.startOffsets = startOffsets;+ this.lengths = lengths;+ this.payloadIndex = payloadIndex;+ this.payloads = payloads;+ this.in = in;+ startPos = in.getPosition();+ reset();+ }++ void reset() {+ term.length = 0;+ in.setPosition(startPos);+ ord = -1;+ }++ @Override+ public BytesRef next() throws IOException {+ if (ord == numTerms - 1) {+ return null;+ } else {+ assert ord < numTerms;+ ++ord;+ }++ // read term+ term.offset = 0;+ term.length = prefixLengths[ord] + suffixLengths[ord];+ if (term.length > term.bytes.length) {+ term.bytes = ArrayUtil.grow(term.bytes, term.length);+ }+ in.readBytes(term.bytes, prefixLengths[ord], suffixLengths[ord]);++ return term;+ }++ @Override+ public SeekStatus seekCeil(BytesRef text) throws IOException {+ if (ord < numTerms && ord >= 0) {+ final int cmp = term().compareTo(text);+ if (cmp == 0) {+ return SeekStatus.FOUND;+ } else if (cmp > 0) {+ reset();+ }+ }+ // linear scan+ while (true) {+ final BytesRef term = next();+ if (term == null) {+ return SeekStatus.END;+ }+ final int cmp = term.compareTo(text);+ if (cmp > 0) {+ return SeekStatus.NOT_FOUND;+ } else if (cmp == 0) {+ return SeekStatus.FOUND;+ }+ }+ }++ @Override+ public void seekExact(long ord) throws IOException {+ throw new UnsupportedOperationException();+ }++ @Override+ public BytesRef term() throws IOException {+ return term;+ }++ @Override+ public long ord() throws IOException {+ throw new UnsupportedOperationException();+ }++ @Override+ public int docFreq() throws IOException {+ return 1;+ }++ @Override+ public long totalTermFreq() throws IOException {+ return termFreqs[ord];+ }++ @Override+ public final PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException {+ final TVPostingsEnum docsEnum;+ if (reuse != null && reuse instanceof TVPostingsEnum) {+ docsEnum = (TVPostingsEnum) reuse;+ } else {+ docsEnum = new TVPostingsEnum();+ }++ docsEnum.reset(+ termFreqs[ord],+ positionIndex[ord],+ positions,+ startOffsets,+ lengths,+ payloads,+ payloadIndex);+ return docsEnum;+ }++ @Override+ public ImpactsEnum impacts(int flags) throws IOException {+ final PostingsEnum delegate = postings(null, PostingsEnum.FREQS);+ return new SlowImpactsEnum(delegate);+ }+ }++ private static class TVPostingsEnum extends PostingsEnum {++ private int doc = -1;+ private int termFreq;+ private int positionIndex;+ private int[] positions;+ private int[] startOffsets;+ private int[] lengths;+ private final BytesRef payload;+ private int[] payloadIndex;+ private int basePayloadOffset;+ private int i;++ TVPostingsEnum() {+ payload = new BytesRef();+ }++ public void reset(+ int freq,+ int positionIndex,+ int[] positions,+ int[] startOffsets,+ int[] lengths,+ BytesRef payloads,+ int[] payloadIndex) {+ this.termFreq = freq;+ this.positionIndex = positionIndex;+ this.positions = positions;+ this.startOffsets = startOffsets;+ this.lengths = lengths;+ this.basePayloadOffset = payloads.offset;+ this.payload.bytes = payloads.bytes;+ payload.offset = payload.length = 0;+ this.payloadIndex = payloadIndex;++ doc = i = -1;+ }++ private void checkDoc() {+ if (doc == NO_MORE_DOCS) {+ throw new IllegalStateException("DocsEnum exhausted");+ } else if (doc == -1) {+ throw new IllegalStateException("DocsEnum not started");+ }+ }++ private void checkPosition() {+ checkDoc();+ if (i < 0) {+ throw new IllegalStateException("Position enum not started");+ } else if (i >= termFreq) {+ throw new IllegalStateException("Read past last position");+ }+ }++ @Override+ public int nextPosition() throws IOException {+ if (doc != 0) {+ throw new IllegalStateException();+ } else if (i >= termFreq - 1) {+ throw new IllegalStateException("Read past last position");+ }++ ++i;++ if (payloadIndex != null) {+ payload.offset = basePayloadOffset + payloadIndex[positionIndex + i];+ payload.length = payloadIndex[positionIndex + i + 1] - payloadIndex[positionIndex + i];+ }++ if (positions == null) {+ return -1;+ } else {+ return positions[positionIndex + i];+ }+ }++ @Override+ public int startOffset() throws IOException {+ checkPosition();+ if (startOffsets == null) {+ return -1;+ } else {+ return startOffsets[positionIndex + i];+ }+ }++ @Override+ public int endOffset() throws IOException {+ checkPosition();+ if (startOffsets == null) {+ return -1;+ } else {+ return startOffsets[positionIndex + i] + lengths[positionIndex + i];+ }+ }++ @Override+ public BytesRef getPayload() throws IOException {+ checkPosition();+ if (payloadIndex == null || payload.length == 0) {+ return null;+ } else {+ return payload;+ }+ }++ @Override+ public int freq() throws IOException {+ checkDoc();+ return termFreq;+ }++ @Override+ public int docID() {+ return doc;+ }++ @Override+ public int nextDoc() throws IOException {+ if (doc == -1) {+ return (doc = 0);+ } else {+ return (doc = NO_MORE_DOCS);+ }+ }++ @Override+ public int advance(int target) throws IOException {+ return slowAdvance(target);+ }++ @Override+ public long cost() {+ return 1;+ }+ }++ private static int sum(int[] arr) {+ int sum = 0;+ for (int el : arr) {+ sum += el;+ }+ return sum;+ }++ @Override+ public long ramBytesUsed() {+ return indexReader.ramBytesUsed();+ }++ @Override+ public Collection<Accountable> getChildResources() {+ return Collections.singleton(Accountables.namedAccountable("term vector index", indexReader));+ }++ @Override+ public void checkIntegrity() throws IOException {+ indexReader.checkIntegrity();+ CodecUtil.checksumEntireFile(vectorsStream);+ }++ @Override+ public String toString() {+ return getClass().getSimpleName()+ + "(mode="+ + compressionMode+ + ",chunksize="+ + chunkSize+ + ")";+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/Lucene90CompressingTermVectorsWriter.javasimilarity index 97%rename from lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.javarename to lucene/core/src/java/org/apache/lucene/codecs/compressing/Lucene90CompressingTermVectorsWriter.javaindex 95c633e1014..4fc88c362c6 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/Lucene90CompressingTermVectorsWriter.java@@ -51,11 +51,11 @@ import org.apache.lucene.util.packed.BlockPackedWriter;import org.apache.lucene.util.packed.PackedInts;/**- * {@link TermVectorsWriter} for {@link CompressingTermVectorsFormat}.+ * {@link TermVectorsWriter} for {@link Lucene90CompressingTermVectorsFormat}.** @lucene.experimental*/-public final class CompressingTermVectorsWriter extends TermVectorsWriter {+public final class Lucene90CompressingTermVectorsWriter extends TermVectorsWriter {// hard limit on the maximum number of documents per chunkstatic final int MAX_DOCUMENTS_PER_CHUNK = 128;@@ -63,14 +63,10 @@ public final class CompressingTermVectorsWriter extends TermVectorsWriter {static final String VECTORS_EXTENSION = "tvd";static final String VECTORS_INDEX_EXTENSION = "tvx";static final String VECTORS_META_EXTENSION = "tvm";- static final String VECTORS_INDEX_CODEC_NAME = "Lucene85TermVectorsIndex";+ static final String VECTORS_INDEX_CODEC_NAME = "Lucene90TermVectorsIndex";- static final int VERSION_START = 1;- static final int VERSION_OFFHEAP_INDEX = 2;- /** Version where all metadata were moved to the meta file. */- static final int VERSION_META = 3;-- static final int VERSION_CURRENT = VERSION_META;+ static final int VERSION_START = 0;+ static final int VERSION_CURRENT = VERSION_START;static final int META_VERSION_START = 0;static final int PACKED_BLOCK_SIZE = 64;@@ -227,7 +223,7 @@ public final class CompressingTermVectorsWriter extends TermVectorsWriter {private final BlockPackedWriter writer;/** Sole constructor. */- CompressingTermVectorsWriter(+ Lucene90CompressingTermVectorsWriter(Directory directory,SegmentInfo si,String segmentSuffix,@@ -792,7 +788,7 @@ public final class CompressingTermVectorsWriter extends TermVectorsWriter {// we try to be extra safe with this impl, but add an escape hatch to// have a workaround for undiscovered bugs.static final String BULK_MERGE_ENABLED_SYSPROP =- CompressingTermVectorsWriter.class.getName() + ".enableBulkMerge";+ Lucene90CompressingTermVectorsWriter.class.getName() + ".enableBulkMerge";static final boolean BULK_MERGE_ENABLED;static {@@ -818,12 +814,13 @@ public final class CompressingTermVectorsWriter extends TermVectorsWriter {MatchingReaders matching = new MatchingReaders(mergeState);for (int readerIndex = 0; readerIndex < numReaders; readerIndex++) {- CompressingTermVectorsReader matchingVectorsReader = null;+ Lucene90CompressingTermVectorsReader matchingVectorsReader = null;final TermVectorsReader vectorsReader = mergeState.termVectorsReaders[readerIndex];if (matching.matchingReaders[readerIndex]) {// we can only bulk-copy if the matching reader is also a CompressingTermVectorsReader- if (vectorsReader != null && vectorsReader instanceof CompressingTermVectorsReader) {- matchingVectorsReader = (CompressingTermVectorsReader) vectorsReader;+ if (vectorsReader != null+ && vectorsReader instanceof Lucene90CompressingTermVectorsReader) {+ matchingVectorsReader = (Lucene90CompressingTermVectorsReader) vectorsReader;}}@@ -936,7 +933,7 @@ public final class CompressingTermVectorsWriter extends TermVectorsWriter {* some worst-case situations (e.g. frequent reopen with tiny flushes), over time the compression* ratio can degrade. This is a safety switch.*/- boolean tooDirty(CompressingTermVectorsReader candidate) {+ boolean tooDirty(Lucene90CompressingTermVectorsReader candidate) {// more than 1% dirty, or more than hard limit of 1024 dirty chunksreturn candidate.getNumDirtyChunks() > 1024|| candidate.getNumDirtyDocs() * 100 > candidate.getNumDocs();diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/MatchingReaders.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/MatchingReaders.javaindex 8ec245a2688..0305ff2a7a7 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/MatchingReaders.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/MatchingReaders.java@@ -23,19 +23,21 @@ import org.apache.lucene.index.SegmentReader;/*** Computes which segments have identical field name to number mappings, which allows stored fields* and term vectors in this codec to be bulk-merged.+ *+ * @lucene.internal*/-class MatchingReaders {+public class MatchingReaders {/*** {@link SegmentReader}s that have identical field name/number mapping, so their stored fields* and term vectors may be bulk merged.*/- final boolean[] matchingReaders;+ public final boolean[] matchingReaders;/** How many {@link #matchingReaders} are set. */final int count;- MatchingReaders(MergeState mergeState) {+ public MatchingReaders(MergeState mergeState) {// If the i'th reader is a SegmentReader and has// identical fieldName -> number mapping, then this// array will be non-null at position i:diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90Codec.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90Codec.javaindex 770f3528522..616d7d71b95 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90Codec.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90Codec.java@@ -30,7 +30,6 @@ import org.apache.lucene.codecs.SegmentInfoFormat;import org.apache.lucene.codecs.StoredFieldsFormat;import org.apache.lucene.codecs.TermVectorsFormat;import org.apache.lucene.codecs.VectorFormat;-import org.apache.lucene.codecs.lucene50.Lucene50TermVectorsFormat;import org.apache.lucene.codecs.lucene86.Lucene86PointsFormat;import org.apache.lucene.codecs.lucene86.Lucene86SegmentInfoFormat;import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat;@@ -65,7 +64,7 @@ public class Lucene90Codec extends Codec {}}- private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat();+ private final TermVectorsFormat vectorsFormat = new Lucene90TermVectorsFormat();private final FieldInfosFormat fieldInfosFormat = new Lucene90FieldInfosFormat();private final SegmentInfoFormat segmentInfosFormat = new Lucene86SegmentInfoFormat();private final LiveDocsFormat liveDocsFormat = new Lucene90LiveDocsFormat();diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90TermVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90TermVectorsFormat.javanew file mode 100644index 00000000000..8c894edcffe--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90TermVectorsFormat.java@@ -0,0 +1,160 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene90;++import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.codecs.TermVectorsFormat;+import org.apache.lucene.codecs.compressing.CompressionMode;+import org.apache.lucene.codecs.compressing.FieldsIndexWriter;+import org.apache.lucene.codecs.compressing.Lucene90CompressingTermVectorsFormat;+import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat;+import org.apache.lucene.store.DataOutput;+import org.apache.lucene.util.packed.BlockPackedWriter;+import org.apache.lucene.util.packed.PackedInts;++/**+ * Lucene 9.0 {@link TermVectorsFormat term vectors format}.+ *+ * <p>Very similarly to {@link Lucene87StoredFieldsFormat}, this format is based on compressed+ * chunks of data, with document-level granularity so that a document can never span across distinct+ * chunks. Moreover, data is made as compact as possible:+ *+ * <ul>+ * <li>textual data is compressed using the very light, <a+ * href="http://code.google.com/p/lz4/">LZ4</a> compression algorithm,+ * <li>binary data is written using fixed-size blocks of {@link PackedInts packed ints}.+ * </ul>+ *+ * <p>Term vectors are stored using two files+ *+ * <ul>+ * <li>a data file where terms, frequencies, positions, offsets and payloads are stored,+ * <li>an index file, loaded into memory, used to locate specific documents in the data file.+ * </ul>+ *+ * Looking up term vectors for any document requires at most 1 disk seek.+ *+ * <p><b>File formats</b>+ *+ * <ol>+ * <li><a id="vector_data"></a>+ * <p>A vector data file (extension <code>.tvd</code>). This file stores terms, frequencies,+ * positions, offsets and payloads for every document. Upon writing a new segment, it+ * accumulates data into memory until the buffer used to store terms and payloads grows beyond+ * 4KB. Then it flushes all metadata, terms and positions to disk using <a+ * href="http://code.google.com/p/lz4/">LZ4</a> compression for terms and payloads and {@link+ * BlockPackedWriter blocks of packed ints} for positions.+ * <p>Here is a more detailed description of the field data file format:+ * <ul>+ * <li>VectorData (.tvd) --> <Header>, PackedIntsVersion, ChunkSize,+ * <Chunk><sup>ChunkCount</sup>, ChunkCount, DirtyChunkCount, Footer+ * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}+ * <li>PackedIntsVersion --> {@link PackedInts#VERSION_CURRENT} as a {@link+ * DataOutput#writeVInt VInt}+ * <li>ChunkSize is the number of bytes of terms to accumulate before flushing, as a {@link+ * DataOutput#writeVInt VInt}+ * <li>ChunkCount is not known in advance and is the number of chunks necessary to store all+ * document of the segment+ * <li>Chunk --> DocBase, ChunkDocs, < NumFields >, < FieldNums >, <+ * FieldNumOffs >, < Flags >, < NumTerms >, < TermLengths >, <+ * TermFreqs >, < Positions >, < StartOffsets >, < Lengths >, <+ * PayloadLengths >, < TermAndPayloads >+ * <li>DocBase is the ID of the first doc of the chunk as a {@link DataOutput#writeVInt+ * VInt}+ * <li>ChunkDocs is the number of documents in the chunk+ * <li>NumFields --> DocNumFields<sup>ChunkDocs</sup>+ * <li>DocNumFields is the number of fields for each doc, written as a {@link+ * DataOutput#writeVInt VInt} if ChunkDocs==1 and as a {@link PackedInts} array+ * otherwise+ * <li>FieldNums --> FieldNumDelta<sup>TotalDistincFields</sup>, a delta-encoded list of+ * the sorted unique field numbers present in the chunk+ * <li>FieldNumOffs --> FieldNumOff<sup>TotalFields</sup>, as a {@link PackedInts} array+ * <li>FieldNumOff is the offset of the field number in FieldNums+ * <li>TotalFields is the total number of fields (sum of the values of NumFields)+ * <li>Flags --> Bit < FieldFlags >+ * <li>Bit is a single bit which when true means that fields have the same options for every+ * document in the chunk+ * <li>FieldFlags --> if Bit==1: Flag<sup>TotalDistinctFields</sup> else+ * Flag<sup>TotalFields</sup>+ * <li>Flag: a 3-bits int where:+ * <ul>+ * <li>the first bit means that the field has positions+ * <li>the second bit means that the field has offsets+ * <li>the third bit means that the field has payloads+ * </ul>+ * <li>NumTerms --> FieldNumTerms<sup>TotalFields</sup>+ * <li>FieldNumTerms: the number of terms for each field, using {@link BlockPackedWriter+ * blocks of 64 packed ints}+ * <li>TermLengths --> PrefixLength<sup>TotalTerms</sup>+ * SuffixLength<sup>TotalTerms</sup>+ * <li>TotalTerms: total number of terms (sum of NumTerms)+ * <li>PrefixLength: 0 for the first term of a field, the common prefix with the previous+ * term otherwise using {@link BlockPackedWriter blocks of 64 packed ints}+ * <li>SuffixLength: length of the term minus PrefixLength for every term using {@link+ * BlockPackedWriter blocks of 64 packed ints}+ * <li>TermFreqs --> TermFreqMinus1<sup>TotalTerms</sup>+ * <li>TermFreqMinus1: (frequency - 1) for each term using {@link BlockPackedWriter blocks+ * of 64 packed ints}+ * <li>Positions --> PositionDelta<sup>TotalPositions</sup>+ * <li>TotalPositions is the sum of frequencies of terms of all fields that have positions+ * <li>PositionDelta: the absolute position for the first position of a term, and the+ * difference with the previous positions for following positions using {@link+ * BlockPackedWriter blocks of 64 packed ints}+ * <li>StartOffsets --> (AvgCharsPerTerm<sup>TotalDistinctFields</sup>)+ * StartOffsetDelta<sup>TotalOffsets</sup>+ * <li>TotalOffsets is the sum of frequencies of terms of all fields that have offsets+ * <li>AvgCharsPerTerm: average number of chars per term, encoded as a float on 4 bytes.+ * They are not present if no field has both positions and offsets enabled.+ * <li>StartOffsetDelta: (startOffset - previousStartOffset - AvgCharsPerTerm *+ * PositionDelta). previousStartOffset is 0 for the first offset and AvgCharsPerTerm is+ * 0 if the field has no positions using {@link BlockPackedWriter blocks of 64 packed+ * ints}+ * <li>Lengths --> LengthMinusTermLength<sup>TotalOffsets</sup>+ * <li>LengthMinusTermLength: (endOffset - startOffset - termLength) using {@link+ * BlockPackedWriter blocks of 64 packed ints}+ * <li>PayloadLengths --> PayloadLength<sup>TotalPayloads</sup>+ * <li>TotalPayloads is the sum of frequencies of terms of all fields that have payloads+ * <li>PayloadLength is the payload length encoded using {@link BlockPackedWriter blocks of+ * 64 packed ints}+ * <li>TermAndPayloads --> LZ4-compressed representation of < FieldTermsAndPayLoads+ * ><sup>TotalFields</sup>+ * <li>FieldTermsAndPayLoads --> Terms (Payloads)+ * <li>Terms: term bytes+ * <li>Payloads: payload bytes (if the field has payloads)+ * <li>ChunkCount --> the number of chunks in this file+ * <li>DirtyChunkCount --> the number of prematurely flushed chunks in this file+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}+ * </ul>+ * <li><a id="vector_index"></a>+ * <p>An index file (extension <code>.tvx</code>).+ * <ul>+ * <li>VectorIndex (.tvx) --> <Header>, <ChunkIndex>, Footer+ * <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}+ * <li>ChunkIndex: See {@link FieldsIndexWriter}+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}+ * </ul>+ * </ol>+ *+ * @lucene.experimental+ */+public final class Lucene90TermVectorsFormat extends Lucene90CompressingTermVectorsFormat {++ /** Sole constructor. */+ public Lucene90TermVectorsFormat() {+ super("Lucene90TermVectorsData", "", CompressionMode.FAST, 1 << 12, 10);+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/package-info.javaindex 83a3912790d..dcab523beb5 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/package-info.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/package-info.java@@ -165,7 +165,7 @@* <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For* each field in each document, a value is stored that is multiplied into the score for hits* on that field.- * <li>{@link org.apache.lucene.codecs.lucene50.Lucene50TermVectorsFormat Term Vectors}. For each+ * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat Term Vectors}. For each* field in each document, the term vector (sometimes called document vector) may be stored. A* term vector consists of term text and term frequency. To add Term Vectors to your index see* the {@link org.apache.lucene.document.Field Field} constructors@@ -290,12 +290,12 @@* <td>Encodes additional scoring factors or other per-document information.</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene50.Lucene50TermVectorsFormat Term Vector Index}</td>+ * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat Term Vector Index}</td>* <td>.tvx</td>* <td>Stores offset into the document data file</td>* </tr>* <tr>- * <td>{@link org.apache.lucene.codecs.lucene50.Lucene50TermVectorsFormat Term Vector Data}</td>+ * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat Term Vector Data}</td>* <td>.tvd</td>* <td>Contains term vector data.</td>* </tr>diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingTermVectorsConsumer.java b/lucene/core/src/java/org/apache/lucene/index/SortingTermVectorsConsumer.javaindex 2c859c9c56b..d6b72f85b1f 100644--- a/lucene/core/src/java/org/apache/lucene/index/SortingTermVectorsConsumer.java+++ b/lucene/core/src/java/org/apache/lucene/index/SortingTermVectorsConsumer.java@@ -24,7 +24,7 @@ import org.apache.lucene.codecs.NormsProducer;import org.apache.lucene.codecs.TermVectorsFormat;import org.apache.lucene.codecs.TermVectorsReader;import org.apache.lucene.codecs.TermVectorsWriter;-import org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat;+import org.apache.lucene.codecs.compressing.Lucene90CompressingTermVectorsFormat;import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.store.Directory;import org.apache.lucene.store.FlushInfo;@@ -37,7 +37,7 @@ import org.apache.lucene.util.IntBlockPool;final class SortingTermVectorsConsumer extends TermVectorsConsumer {private static final TermVectorsFormat TEMP_TERM_VECTORS_FORMAT =- new CompressingTermVectorsFormat(+ new Lucene90CompressingTermVectorsFormat("TempTermVectors", "", SortingStoredFieldsConsumer.NO_COMPRESSION, 8 * 1024, 10);TrackingTmpOutputDirectoryWrapper tmpDirectory;diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90TermVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90TermVectorsFormat.javanew file mode 100644index 00000000000..00bf963ab03--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90TermVectorsFormat.java@@ -0,0 +1,28 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene90;++import org.apache.lucene.codecs.Codec;+import org.apache.lucene.index.BaseTermVectorsFormatTestCase;+import org.apache.lucene.util.TestUtil;++public class TestLucene90TermVectorsFormat extends BaseTermVectorsFormatTestCase {+ @Override+ protected Codec getCodec() {+ return TestUtil.getDefaultCodec();+ }+}diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.javaindex 7539511a250..cc76011a0b4 100644--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java+++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java@@ -102,7 +102,7 @@ public abstract class CompressingCodec extends FilterCodec {}private final CompressingStoredFieldsFormat storedFieldsFormat;- private final CompressingTermVectorsFormat termVectorsFormat;+ private final Lucene90CompressingTermVectorsFormat termVectorsFormat;/** Creates a compressing codec with a given segment suffix */public CompressingCodec(@@ -117,7 +117,7 @@ public abstract class CompressingCodec extends FilterCodec {new CompressingStoredFieldsFormat(name, segmentSuffix, compressionMode, chunkSize, maxDocsPerChunk, blockShift);this.termVectorsFormat =- new CompressingTermVectorsFormat(+ new Lucene90CompressingTermVectorsFormat(name, segmentSuffix, compressionMode, chunkSize, blockShift);}diff --git a/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingTermVectorsFormat.java b/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingTermVectorsFormat.javaindex 18785f04106..1c8f1d00e9e 100644--- a/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingTermVectorsFormat.java+++ b/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingTermVectorsFormat.java@@ -100,8 +100,8 @@ public class TestCompressingTermVectorsFormat extends BaseTermVectorsFormatTestC// examine dirty counts:for (LeafReaderContext leaf : ir2.leaves()) {CodecReader sr = (CodecReader) leaf.reader();- CompressingTermVectorsReader reader =- (CompressingTermVectorsReader) sr.getTermVectorsReader();+ Lucene90CompressingTermVectorsReader reader =+ (Lucene90CompressingTermVectorsReader) sr.getTermVectorsReader();assertTrue(reader.getNumDirtyDocs() > 0);assertEquals(1, reader.getNumDirtyChunks());}@@ -113,7 +113,8 @@ public class TestCompressingTermVectorsFormat extends BaseTermVectorsFormatTestCir.close();ir = ir2;CodecReader sr = (CodecReader) getOnlyLeafReader(ir);- CompressingTermVectorsReader reader = (CompressingTermVectorsReader) sr.getTermVectorsReader();+ Lucene90CompressingTermVectorsReader reader =+ (Lucene90CompressingTermVectorsReader) sr.getTermVectorsReader();// we could get lucky, and have zero, but typically one.assertTrue(reader.getNumDirtyChunks() <= 1);ir.close();