Apache Solr open-source search software
LUCENE-9484: Allow sorting an index after the fact (#1789)
Simon Willnauer430a43520aea
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex a0435b2bca6..830ad1ba4c1 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -182,6 +182,10 @@ New Featuressmall segments on getReader, subject to a configurable timeout, to improvesearch performance by reducing the number of small segments for searching. (Simon Willnauer)+* LUCENE-9484: Allow sorting an index after it was created. With SortingCodecReader, existing+ unsorted segments can be wrapped and merged into a fresh index using IndexWriter#addIndices+ API. (Simon Willnauer, Adrien Grand)+Improvements---------------------diff --git a/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.javaindex e213a485ea0..775f32508a0 100644--- a/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java+++ b/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java@@ -24,6 +24,7 @@ import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.store.DataInput;import org.apache.lucene.store.DataOutput;import org.apache.lucene.util.ArrayUtil;+import org.apache.lucene.util.BitSet;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.BytesRefBuilder;import org.apache.lucene.util.Counter;@@ -99,7 +100,7 @@ class BinaryDocValuesWriter extends DocValuesWriter<BinaryDocValues> {bytesUsed = newBytesUsed;}- private SortingLeafReader.CachedBinaryDVs sortDocValues(int maxDoc, Sorter.DocMap sortMap, BinaryDocValues oldValues) throws IOException {+ static CachedBinaryDVs sortDocValues(int maxDoc, Sorter.DocMap sortMap, BinaryDocValues oldValues) throws IOException {FixedBitSet docsWithField = new FixedBitSet(maxDoc);BytesRef[] values = new BytesRef[maxDoc];while (true) {@@ -111,7 +112,7 @@ class BinaryDocValuesWriter extends DocValuesWriter<BinaryDocValues> {docsWithField.set(newDocID);values[newDocID] = BytesRef.deepCopyOf(oldValues.binaryValue());}- return new SortingLeafReader.CachedBinaryDVs(values, docsWithField);+ return new CachedBinaryDVs(values, docsWithField);}@Override@@ -128,7 +129,7 @@ class BinaryDocValuesWriter extends DocValuesWriter<BinaryDocValues> {if (finalLengths == null) {finalLengths = this.lengths.build();}- final SortingLeafReader.CachedBinaryDVs sorted;+ final CachedBinaryDVs sorted;if (sortMap != null) {sorted = sortDocValues(state.segmentInfo.maxDoc(), sortMap,new BufferedBinaryDocValues(finalLengths, maxLength, bytes.getDataInput(), docsWithField.iterator()));@@ -145,7 +146,7 @@ class BinaryDocValuesWriter extends DocValuesWriter<BinaryDocValues> {if (sorted == null) {return new BufferedBinaryDocValues(finalLengths, maxLength, bytes.getDataInput(), docsWithField.iterator());} else {- return new SortingLeafReader.SortingBinaryDocValues(sorted);+ return new SortingBinaryDocValues(sorted);}}});@@ -202,4 +203,63 @@ class BinaryDocValuesWriter extends DocValuesWriter<BinaryDocValues> {return value.get();}}++ static class SortingBinaryDocValues extends BinaryDocValues {+ private final CachedBinaryDVs dvs;+ private int docID = -1;+ private long cost = -1;++ SortingBinaryDocValues(CachedBinaryDVs dvs) {+ this.dvs = dvs;+ }++ @Override+ public int nextDoc() {+ if (docID+1 == dvs.docsWithField.length()) {+ docID = NO_MORE_DOCS;+ } else {+ docID = dvs.docsWithField.nextSetBit(docID+1);+ }+ return docID;+ }++ @Override+ public int docID() {+ return docID;+ }++ @Override+ public int advance(int target) {+ throw new UnsupportedOperationException("use nextDoc instead");+ }++ @Override+ public boolean advanceExact(int target) throws IOException {+ throw new UnsupportedOperationException("use nextDoc instead");+ }++ @Override+ public BytesRef binaryValue() {+ return dvs.values[docID];+ }++ @Override+ public long cost() {+ if (cost == -1) {+ cost = dvs.docsWithField.cardinality();+ }+ return cost;+ }+ }++ static class CachedBinaryDVs {+ // TODO: at least cutover to BytesRefArray here:+ private final BytesRef[] values;+ private final BitSet docsWithField;++ CachedBinaryDVs(BytesRef[] values, BitSet docsWithField) {+ this.values = values;+ this.docsWithField = docsWithField;+ }+ }}diff --git a/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java b/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.javaindex bbc7b18dcad..702218bccee 100644--- a/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java@@ -26,9 +26,16 @@ import java.util.Map;import org.apache.lucene.codecs.FieldsConsumer;import org.apache.lucene.codecs.NormsProducer;import org.apache.lucene.search.DocIdSetIterator;+import org.apache.lucene.store.ByteBuffersDataInput;+import org.apache.lucene.store.ByteBuffersDataOutput;+import org.apache.lucene.store.DataOutput;+import org.apache.lucene.util.ArrayUtil;+import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.CollectionUtil;import org.apache.lucene.util.FixedBitSet;import org.apache.lucene.util.IOUtils;+import org.apache.lucene.util.TimSorter;+import org.apache.lucene.util.automaton.CompiledAutomaton;final class FreqProxTermsWriter extends TermsHash {@@ -88,7 +95,20 @@ final class FreqProxTermsWriter extends TermsHash {Fields fields = new FreqProxFields(allFields);applyDeletes(state, fields);if (sortMap != null) {- fields = new SortingLeafReader.SortingFields(fields, state.fieldInfos, sortMap);+ final Sorter.DocMap docMap = sortMap;+ final FieldInfos infos = state.fieldInfos;+ fields = new FilterLeafReader.FilterFields(fields) {++ @Override+ public Terms terms(final String field) throws IOException {+ Terms terms = in.terms(field);+ if (terms == null) {+ return null;+ } else {+ return new SortingTerms(terms, infos.fieldInfo(field).getIndexOptions(), docMap);+ }+ }+ };}FieldsConsumer consumer = state.segmentInfo.getCodec().postingsFormat().fieldsConsumer(state);@@ -110,4 +130,503 @@ final class FreqProxTermsWriter extends TermsHash {public TermsHashPerField addField(FieldInvertState invertState, FieldInfo fieldInfo) {return new FreqProxTermsWriterPerField(invertState, this, fieldInfo, nextTermsHash.addField(invertState, fieldInfo));}++ static class SortingTerms extends FilterLeafReader.FilterTerms {++ private final Sorter.DocMap docMap;+ private final IndexOptions indexOptions;++ SortingTerms(final Terms in, IndexOptions indexOptions, final Sorter.DocMap docMap) {+ super(in);+ this.docMap = docMap;+ this.indexOptions = indexOptions;+ }++ @Override+ public TermsEnum iterator() throws IOException {+ return new SortingTermsEnum(in.iterator(), docMap, indexOptions, hasPositions());+ }++ @Override+ public TermsEnum intersect(CompiledAutomaton compiled, BytesRef startTerm)+ throws IOException {+ return new SortingTermsEnum(in.intersect(compiled, startTerm), docMap, indexOptions, hasPositions());+ }++ }++ private static class SortingTermsEnum extends FilterLeafReader.FilterTermsEnum {++ final Sorter.DocMap docMap; // pkg-protected to avoid synthetic accessor methods+ private final IndexOptions indexOptions;+ private final boolean hasPositions;++ SortingTermsEnum(final TermsEnum in, Sorter.DocMap docMap, IndexOptions indexOptions, boolean hasPositions) {+ super(in);+ this.docMap = docMap;+ this.indexOptions = indexOptions;+ this.hasPositions = hasPositions;+ }++ @Override+ public PostingsEnum postings( PostingsEnum reuse, final int flags) throws IOException {++ if (hasPositions && PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)) {+ final PostingsEnum inReuse;+ final SortingPostingsEnum wrapReuse;+ if (reuse != null && reuse instanceof SortingPostingsEnum) {+ // if we're asked to reuse the given DocsEnum and it is Sorting, return+ // the wrapped one, since some Codecs expect it.+ wrapReuse = (SortingPostingsEnum) reuse;+ inReuse = wrapReuse.getWrapped();+ } else {+ wrapReuse = null;+ inReuse = reuse;+ }++ final PostingsEnum inDocsAndPositions = in.postings(inReuse, flags);+ // we ignore the fact that offsets may be stored but not asked for,+ // since this code is expected to be used during addIndexes which will+ // ask for everything. if that assumption changes in the future, we can+ // factor in whether 'flags' says offsets are not required.+ final boolean storeOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;+ return new SortingPostingsEnum(docMap.size(), wrapReuse, inDocsAndPositions, docMap, storeOffsets);+ }++ final PostingsEnum inReuse;+ final SortingDocsEnum wrapReuse;+ if (reuse != null && reuse instanceof SortingDocsEnum) {+ // if we're asked to reuse the given DocsEnum and it is Sorting, return+ // the wrapped one, since some Codecs expect it.+ wrapReuse = (SortingDocsEnum) reuse;+ inReuse = wrapReuse.getWrapped();+ } else {+ wrapReuse = null;+ inReuse = reuse;+ }++ final PostingsEnum inDocs = in.postings(inReuse, flags);+ final boolean withFreqs = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >=0 && PostingsEnum.featureRequested(flags, PostingsEnum.FREQS);+ return new SortingDocsEnum(docMap.size(), wrapReuse, inDocs, withFreqs, docMap);+ }++ }++ static class SortingDocsEnum extends FilterLeafReader.FilterPostingsEnum {++ private static final class DocFreqSorter extends TimSorter {++ private int[] docs;+ private int[] freqs;+ private final int[] tmpDocs;+ private int[] tmpFreqs;++ DocFreqSorter(int maxDoc) {+ super(maxDoc / 64);+ this.tmpDocs = new int[maxDoc / 64];+ }++ public void reset(int[] docs, int[] freqs) {+ this.docs = docs;+ this.freqs = freqs;+ if (freqs != null && tmpFreqs == null) {+ tmpFreqs = new int[tmpDocs.length];+ }+ }++ @Override+ protected int compare(int i, int j) {+ return docs[i] - docs[j];+ }++ @Override+ protected void swap(int i, int j) {+ int tmpDoc = docs[i];+ docs[i] = docs[j];+ docs[j] = tmpDoc;++ if (freqs != null) {+ int tmpFreq = freqs[i];+ freqs[i] = freqs[j];+ freqs[j] = tmpFreq;+ }+ }++ @Override+ protected void copy(int src, int dest) {+ docs[dest] = docs[src];+ if (freqs != null) {+ freqs[dest] = freqs[src];+ }+ }++ @Override+ protected void save(int i, int len) {+ System.arraycopy(docs, i, tmpDocs, 0, len);+ if (freqs != null) {+ System.arraycopy(freqs, i, tmpFreqs, 0, len);+ }+ }++ @Override+ protected void restore(int i, int j) {+ docs[j] = tmpDocs[i];+ if (freqs != null) {+ freqs[j] = tmpFreqs[i];+ }+ }++ @Override+ protected int compareSaved(int i, int j) {+ return tmpDocs[i] - docs[j];+ }+ }++ private final int maxDoc;+ private final DocFreqSorter sorter;+ private int[] docs;+ private int[] freqs;+ private int docIt = -1;+ private final int upto;+ private final boolean withFreqs;++ SortingDocsEnum(int maxDoc, SortingDocsEnum reuse, final PostingsEnum in, boolean withFreqs, final Sorter.DocMap docMap) throws IOException {+ super(in);+ this.maxDoc = maxDoc;+ this.withFreqs = withFreqs;+ if (reuse != null) {+ if (reuse.maxDoc == maxDoc) {+ sorter = reuse.sorter;+ } else {+ sorter = new DocFreqSorter(maxDoc);+ }+ docs = reuse.docs;+ freqs = reuse.freqs; // maybe null+ } else {+ docs = new int[64];+ sorter = new DocFreqSorter(maxDoc);+ }+ docIt = -1;+ int i = 0;+ int doc;+ if (withFreqs) {+ if (freqs == null || freqs.length < docs.length) {+ freqs = new int[docs.length];+ }+ while ((doc = in.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS){+ if (i >= docs.length) {+ docs = ArrayUtil.grow(docs, docs.length + 1);+ freqs = ArrayUtil.grow(freqs, freqs.length + 1);+ }+ docs[i] = docMap.oldToNew(doc);+ freqs[i] = in.freq();+ ++i;+ }+ } else {+ freqs = null;+ while ((doc = in.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS){+ if (i >= docs.length) {+ docs = ArrayUtil.grow(docs, docs.length + 1);+ }+ docs[i++] = docMap.oldToNew(doc);+ }+ }+ // TimSort can save much time compared to other sorts in case of+ // reverse sorting, or when sorting a concatenation of sorted readers+ sorter.reset(docs, freqs);+ sorter.sort(0, i);+ upto = i;+ }++ // for testing+ boolean reused(PostingsEnum other) {+ if (other == null || !(other instanceof SortingDocsEnum)) {+ return false;+ }+ return docs == ((SortingDocsEnum) other).docs;+ }++ @Override+ public int advance(final int target) throws IOException {+ // need to support it for checkIndex, but in practice it won't be called, so+ // don't bother to implement efficiently for now.+ return slowAdvance(target);+ }++ @Override+ public int docID() {+ return docIt < 0 ? -1 : docIt >= upto ? NO_MORE_DOCS : docs[docIt];+ }++ @Override+ public int freq() throws IOException {+ return withFreqs && docIt < upto ? freqs[docIt] : 1;+ }++ @Override+ public int nextDoc() throws IOException {+ if (++docIt >= upto) return NO_MORE_DOCS;+ return docs[docIt];+ }++ /** Returns the wrapped {@link PostingsEnum}. */+ PostingsEnum getWrapped() {+ return in;+ }++ // we buffer up docs/freqs only, don't forward any positions requests to underlying enum++ @Override+ public int nextPosition() throws IOException {+ return -1;+ }++ @Override+ public int startOffset() throws IOException {+ return -1;+ }++ @Override+ public int endOffset() throws IOException {+ return -1;+ }++ @Override+ public BytesRef getPayload() throws IOException {+ return null;+ }+ }++ static class SortingPostingsEnum extends FilterLeafReader.FilterPostingsEnum {++ /**+ * A {@link TimSorter} which sorts two parallel arrays of doc IDs and+ * offsets in one go. Everyti+ * me a doc ID is 'swapped', its corresponding offset+ * is swapped too.+ */+ private static final class DocOffsetSorter extends TimSorter {++ private int[] docs;+ private long[] offsets;+ private final int[] tmpDocs;+ private final long[] tmpOffsets;++ public DocOffsetSorter(int maxDoc) {+ super(maxDoc / 64);+ this.tmpDocs = new int[maxDoc / 64];+ this.tmpOffsets = new long[maxDoc / 64];+ }++ public void reset(int[] docs, long[] offsets) {+ this.docs = docs;+ this.offsets = offsets;+ }++ @Override+ protected int compare(int i, int j) {+ return docs[i] - docs[j];+ }++ @Override+ protected void swap(int i, int j) {+ int tmpDoc = docs[i];+ docs[i] = docs[j];+ docs[j] = tmpDoc;++ long tmpOffset = offsets[i];+ offsets[i] = offsets[j];+ offsets[j] = tmpOffset;+ }++ @Override+ protected void copy(int src, int dest) {+ docs[dest] = docs[src];+ offsets[dest] = offsets[src];+ }++ @Override+ protected void save(int i, int len) {+ System.arraycopy(docs, i, tmpDocs, 0, len);+ System.arraycopy(offsets, i, tmpOffsets, 0, len);+ }++ @Override+ protected void restore(int i, int j) {+ docs[j] = tmpDocs[i];+ offsets[j] = tmpOffsets[i];+ }++ @Override+ protected int compareSaved(int i, int j) {+ return tmpDocs[i] - docs[j];+ }+ }++ private final int maxDoc;+ private final DocOffsetSorter sorter;+ private int[] docs;+ private long[] offsets;+ private final int upto;++ private final ByteBuffersDataInput postingInput;+ private final boolean storeOffsets;++ private int docIt = -1;+ private int pos;+ private int startOffset = -1;+ private int endOffset = -1;+ private final BytesRef payload;+ private int currFreq;++ private final ByteBuffersDataOutput buffer;++ SortingPostingsEnum(int maxDoc, SortingPostingsEnum reuse, final PostingsEnum in, Sorter.DocMap docMap, boolean storeOffsets) throws IOException {+ super(in);+ this.maxDoc = maxDoc;+ this.storeOffsets = storeOffsets;+ if (reuse != null) {+ docs = reuse.docs;+ offsets = reuse.offsets;+ payload = reuse.payload;+ buffer = reuse.buffer;+ buffer.reset();+ if (reuse.maxDoc == maxDoc) {+ sorter = reuse.sorter;+ } else {+ sorter = new DocOffsetSorter(maxDoc);+ }+ } else {+ docs = new int[32];+ offsets = new long[32];+ payload = new BytesRef(32);+ buffer = ByteBuffersDataOutput.newResettableInstance();+ sorter = new DocOffsetSorter(maxDoc);+ }++ int doc;+ int i = 0;+ while ((doc = in.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {+ if (i == docs.length) {+ final int newLength = ArrayUtil.oversize(i + 1, 4);+ docs = ArrayUtil.growExact(docs, newLength);+ offsets = ArrayUtil.growExact(offsets, newLength);+ }+ docs[i] = docMap.oldToNew(doc);+ offsets[i] = buffer.size();+ addPositions(in, buffer);+ i++;+ }+ upto = i;+ sorter.reset(docs, offsets);+ sorter.sort(0, upto);++ this.postingInput = buffer.toDataInput();+ }++ // for testing+ boolean reused(PostingsEnum other) {+ if (other == null || !(other instanceof SortingPostingsEnum)) {+ return false;+ }+ return docs == ((SortingPostingsEnum) other).docs;+ }++ private void addPositions(final PostingsEnum in, final DataOutput out) throws IOException {+ int freq = in.freq();+ out.writeVInt(freq);+ int previousPosition = 0;+ int previousEndOffset = 0;+ for (int i = 0; i < freq; i++) {+ final int pos = in.nextPosition();+ final BytesRef payload = in.getPayload();+ // The low-order bit of token is set only if there is a payload, the+ // previous bits are the delta-encoded position.+ final int token = (pos - previousPosition) << 1 | (payload == null ? 0 : 1);+ out.writeVInt(token);+ previousPosition = pos;+ if (storeOffsets) { // don't encode offsets if they are not stored+ final int startOffset = in.startOffset();+ final int endOffset = in.endOffset();+ out.writeVInt(startOffset - previousEndOffset);+ out.writeVInt(endOffset - startOffset);+ previousEndOffset = endOffset;+ }+ if (payload != null) {+ out.writeVInt(payload.length);+ out.writeBytes(payload.bytes, payload.offset, payload.length);+ }+ }+ }++ @Override+ public int advance(final int target) throws IOException {+ // need to support it for checkIndex, but in practice it won't be called, so+ // don't bother to implement efficiently for now.+ return slowAdvance(target);+ }++ @Override+ public int docID() {+ return docIt < 0 ? -1 : docIt >= upto ? NO_MORE_DOCS : docs[docIt];+ }++ @Override+ public int endOffset() throws IOException {+ return endOffset;+ }++ @Override+ public int freq() throws IOException {+ return currFreq;+ }++ @Override+ public BytesRef getPayload() throws IOException {+ return payload.length == 0 ? null : payload;+ }++ @Override+ public int nextDoc() throws IOException {+ if (++docIt >= upto) return DocIdSetIterator.NO_MORE_DOCS;+ postingInput.seek(offsets[docIt]);+ currFreq = postingInput.readVInt();+ // reset variables used in nextPosition+ pos = 0;+ endOffset = 0;+ return docs[docIt];+ }++ @Override+ public int nextPosition() throws IOException {+ final int token = postingInput.readVInt();+ pos += token >>> 1;+ if (storeOffsets) {+ startOffset = endOffset + postingInput.readVInt();+ endOffset = startOffset + postingInput.readVInt();+ }+ if ((token & 1) != 0) {+ payload.offset = 0;+ payload.length = postingInput.readVInt();+ if (payload.length > payload.bytes.length) {+ payload.bytes = new byte[ArrayUtil.oversize(payload.length, 1)];+ }+ postingInput.readBytes(payload.bytes, 0, payload.length);+ } else {+ payload.length = 0;+ }+ return pos;+ }++ @Override+ public int startOffset() throws IOException {+ return startOffset;+ }++ /** Returns the wrapped {@link PostingsEnum}. */+ PostingsEnum getWrapped() {+ return in;+ }+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/index/NormValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/NormValuesWriter.javaindex 0cfb0815bff..29643524c00 100644--- a/lucene/core/src/java/org/apache/lucene/index/NormValuesWriter.java+++ b/lucene/core/src/java/org/apache/lucene/index/NormValuesWriter.java@@ -70,7 +70,7 @@ class NormValuesWriter {public void flush(SegmentWriteState state, Sorter.DocMap sortMap, NormsConsumer normsConsumer) throws IOException {final PackedLongValues values = pending.build();- final SortingLeafReader.CachedNumericDVs sorted;+ final NumericDocValuesWriter.CachedNumericDVs sorted;if (sortMap != null) {sorted = NumericDocValuesWriter.sortDocValues(state.segmentInfo.maxDoc(), sortMap,new BufferedNorms(values, docsWithField.iterator()));@@ -87,7 +87,7 @@ class NormValuesWriter {if (sorted == null) {return new BufferedNorms(values, docsWithField.iterator());} else {- return new SortingLeafReader.SortingNumericDocValues(sorted);+ return new NumericDocValuesWriter.SortingNumericDocValues(sorted);}}diff --git a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.javaindex 87de98f234d..a696bcaf185 100644--- a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java+++ b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java@@ -21,6 +21,7 @@ import java.io.IOException;import org.apache.lucene.codecs.DocValuesConsumer;import org.apache.lucene.search.DocIdSetIterator;+import org.apache.lucene.util.BitSet;import org.apache.lucene.util.Counter;import org.apache.lucene.util.FixedBitSet;import org.apache.lucene.util.packed.PackedInts;@@ -40,7 +41,7 @@ class NumericDocValuesWriter extends DocValuesWriter<NumericDocValues> {private final FieldInfo fieldInfo;private int lastDocID = -1;- public NumericDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed) {+ NumericDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed) {pending = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT);docsWithField = new DocsWithFieldSet();bytesUsed = pending.ramBytesUsed() + docsWithField.ramBytesUsed();@@ -76,7 +77,7 @@ class NumericDocValuesWriter extends DocValuesWriter<NumericDocValues> {return new BufferedNumericDocValues(finalValues, docsWithField.iterator());}- static SortingLeafReader.CachedNumericDVs sortDocValues(int maxDoc, Sorter.DocMap sortMap, NumericDocValues oldDocValues) throws IOException {+ static CachedNumericDVs sortDocValues(int maxDoc, Sorter.DocMap sortMap, NumericDocValues oldDocValues) throws IOException {FixedBitSet docsWithField = new FixedBitSet(maxDoc);long[] values = new long[maxDoc];while (true) {@@ -88,7 +89,7 @@ class NumericDocValuesWriter extends DocValuesWriter<NumericDocValues> {docsWithField.set(newDocID);values[newDocID] = oldDocValues.longValue();}- return new SortingLeafReader.CachedNumericDVs(values, docsWithField);+ return new CachedNumericDVs(values, docsWithField);}@Override@@ -96,7 +97,7 @@ class NumericDocValuesWriter extends DocValuesWriter<NumericDocValues> {if (finalValues == null) {finalValues = pending.build();}- final SortingLeafReader.CachedNumericDVs sorted;+ final CachedNumericDVs sorted;if (sortMap != null) {NumericDocValues oldValues = new BufferedNumericDocValues(finalValues, docsWithField.iterator());sorted = sortDocValues(state.segmentInfo.maxDoc(), sortMap, oldValues);@@ -114,7 +115,7 @@ class NumericDocValuesWriter extends DocValuesWriter<NumericDocValues> {if (sorted == null) {return new BufferedNumericDocValues(finalValues, docsWithField.iterator());} else {- return new SortingLeafReader.SortingNumericDocValues(sorted);+ return new SortingNumericDocValues(sorted);}}});@@ -165,4 +166,65 @@ class NumericDocValuesWriter extends DocValuesWriter<NumericDocValues> {return value;}}++ static class SortingNumericDocValues extends NumericDocValues {++ private final CachedNumericDVs dvs;+ private int docID = -1;+ private long cost = -1;++ SortingNumericDocValues(CachedNumericDVs dvs) {+ this.dvs = dvs;+ }++ @Override+ public int docID() {+ return docID;+ }++ @Override+ public int nextDoc() {+ if (docID+1 == dvs.docsWithField.length()) {+ docID = NO_MORE_DOCS;+ } else {+ docID = dvs.docsWithField.nextSetBit(docID+1);+ }+ return docID;+ }++ @Override+ public int advance(int target) {+ throw new UnsupportedOperationException("use nextDoc() instead");+ }++ @Override+ public boolean advanceExact(int target) throws IOException {+ // needed in IndexSorter#{Long|Int|Double|Float}Sorter+ docID = target;+ return dvs.docsWithField.get(target);+ }++ @Override+ public long longValue() {+ return dvs.values[docID];+ }++ @Override+ public long cost() {+ if (cost == -1) {+ cost = dvs.docsWithField.cardinality();+ }+ return cost;+ }+ }++ static class CachedNumericDVs {+ private final long[] values;+ private final BitSet docsWithField;++ CachedNumericDVs(long[] values, BitSet docsWithField) {+ this.values = values;+ this.docsWithField = docsWithField;+ }+ }}diff --git a/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.javaindex 2252f003cb2..4ed81f70d7f 100644--- a/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java+++ b/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java@@ -160,7 +160,7 @@ class SortedDocValuesWriter extends DocValuesWriter<SortedDocValues> {if (sorted == null) {return buf;}- return new SortingLeafReader.SortingSortedDocValues(buf, sorted);+ return new SortingSortedDocValues(buf, sorted);}});}@@ -233,4 +233,72 @@ class SortedDocValuesWriter extends DocValuesWriter<SortedDocValues> {}}+ static class SortingSortedDocValues extends SortedDocValues {++ private final SortedDocValues in;+ private final int[] ords;+ private int docID = -1;++ SortingSortedDocValues(SortedDocValues in, int[] ords) {+ this.in = in;+ this.ords = ords;+ assert ords != null;+ }++ @Override+ public int docID() {+ return docID;+ }++ @Override+ public int nextDoc() {+ while (true) {+ docID++;+ if (docID == ords.length) {+ docID = NO_MORE_DOCS;+ break;+ }+ if (ords[docID] != -1) {+ break;+ }+ // skip missing docs+ }++ return docID;+ }++ @Override+ public int advance(int target) {+ throw new UnsupportedOperationException("use nextDoc instead");++ }++ @Override+ public boolean advanceExact(int target) throws IOException {+ // needed in IndexSorter#StringSorter+ docID = target;+ return ords[target] != -1;+ }++ @Override+ public int ordValue() {+ return ords[docID];+ }++ @Override+ public long cost() {+ return in.cost();+ }++ @Override+ public BytesRef lookupOrd(int ord) throws IOException {+ return in.lookupOrd(ord);+ }++ @Override+ public int getValueCount() {+ return in.getValueCount();+ }+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/index/SortedNumericDocValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/SortedNumericDocValuesWriter.javaindex 83c394fc207..0acf93b2014 100644--- a/lucene/core/src/java/org/apache/lucene/index/SortedNumericDocValuesWriter.java+++ b/lucene/core/src/java/org/apache/lucene/index/SortedNumericDocValuesWriter.java@@ -108,7 +108,7 @@ class SortedNumericDocValuesWriter extends DocValuesWriter<SortedNumericDocValuereturn new BufferedSortedNumericDocValues(finalValues, finalValuesCount, docsWithField.iterator());}- private long[][] sortDocValues(int maxDoc, Sorter.DocMap sortMap, SortedNumericDocValues oldValues) throws IOException {+ static long[][] sortDocValues(int maxDoc, Sorter.DocMap sortMap, SortedNumericDocValues oldValues) throws IOException {long[][] values = new long[maxDoc][];int docID;while ((docID = oldValues.nextDoc()) != NO_MORE_DOCS) {@@ -155,7 +155,7 @@ class SortedNumericDocValuesWriter extends DocValuesWriter<SortedNumericDocValueif (sorted == null) {return buf;} else {- return new SortingLeafReader.SortingSortedNumericDocValues(buf, sorted);+ return new SortingSortedNumericDocValues(buf, sorted);}}});@@ -168,7 +168,7 @@ class SortedNumericDocValuesWriter extends DocValuesWriter<SortedNumericDocValueprivate int valueCount;private int valueUpto;- public BufferedSortedNumericDocValues(PackedLongValues values, PackedLongValues valueCounts, DocIdSetIterator docsWithField) {+ BufferedSortedNumericDocValues(PackedLongValues values, PackedLongValues valueCounts, DocIdSetIterator docsWithField) {valuesIter = values.iterator();valueCountsIter = valueCounts.iterator();this.docsWithField = docsWithField;@@ -223,4 +223,68 @@ class SortedNumericDocValuesWriter extends DocValuesWriter<SortedNumericDocValue}}+ static class SortingSortedNumericDocValues extends SortedNumericDocValues {+ private final SortedNumericDocValues in;+ private final long[][] values;+ private int docID = -1;+ private int upto;++ SortingSortedNumericDocValues(SortedNumericDocValues in, long[][] values) {+ this.in = in;+ this.values = values;+ }++ @Override+ public int docID() {+ return docID;+ }++ @Override+ public int nextDoc() {+ while (true) {+ docID++;+ if (docID == values.length) {+ docID = NO_MORE_DOCS;+ break;+ }+ if (values[docID] != null) {+ break;+ }+ // skip missing docs+ }+ upto = 0;+ return docID;+ }++ @Override+ public int advance(int target) {+ throw new UnsupportedOperationException("use nextDoc instead");+ }++ @Override+ public boolean advanceExact(int target) throws IOException {+ docID = target;+ upto = 0;+ return values[docID] != null;+ }++ @Override+ public long nextValue() {+ if (upto == values[docID].length) {+ throw new AssertionError();+ } else {+ return values[docID][upto++];+ }+ }++ @Override+ public long cost() {+ return in.cost();+ }++ @Override+ public int docValueCount() {+ return values[docID].length;+ }+ }}diff --git a/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.javaindex 022b17da095..c444edd1e7e 100644--- a/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java+++ b/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java@@ -139,7 +139,7 @@ class SortedSetDocValuesWriter extends DocValuesWriter<SortedSetDocValues> {bytesUsed = newBytesUsed;}- private long[][] sortDocValues(int maxDoc, Sorter.DocMap sortMap, SortedSetDocValues oldValues) throws IOException {+ static long[][] sortDocValues(int maxDoc, Sorter.DocMap sortMap, SortedSetDocValues oldValues) throws IOException {long[][] ords = new long[maxDoc][];int docID;while ((docID = oldValues.nextDoc()) != NO_MORE_DOCS) {@@ -222,7 +222,7 @@ class SortedSetDocValuesWriter extends DocValuesWriter<SortedSetDocValues> {if (sorted == null) {return buf;} else {- return new SortingLeafReader.SortingSortedSetDocValues(buf, sorted);+ return new SortingSortedSetDocValues(buf, sorted);}}});@@ -241,7 +241,7 @@ class SortedSetDocValuesWriter extends DocValuesWriter<SortedSetDocValues> {private int ordCount;private int ordUpto;- public BufferedSortedSetDocValues(int[] sortedValues, int[] ordMap, BytesRefHash hash, PackedLongValues ords, PackedLongValues ordCounts, int maxCount, DocIdSetIterator docsWithField) {+ BufferedSortedSetDocValues(int[] sortedValues, int[] ordMap, BytesRefHash hash, PackedLongValues ords, PackedLongValues ordCounts, int maxCount, DocIdSetIterator docsWithField) {this.currentDoc = new int[maxCount];this.sortedValues = sortedValues;this.ordMap = ordMap;@@ -308,4 +308,75 @@ class SortedSetDocValuesWriter extends DocValuesWriter<SortedSetDocValues> {}}+ static class SortingSortedSetDocValues extends SortedSetDocValues {++ private final SortedSetDocValues in;+ private final long[][] ords;+ private int docID = -1;+ private int ordUpto;++ SortingSortedSetDocValues(SortedSetDocValues in, long[][] ords) {+ this.in = in;+ this.ords = ords;+ }++ @Override+ public int docID() {+ return docID;+ }++ @Override+ public int nextDoc() {+ while (true) {+ docID++;+ if (docID == ords.length) {+ docID = NO_MORE_DOCS;+ break;+ }+ if (ords[docID] != null) {+ break;+ }+ // skip missing docs+ }+ ordUpto = 0;+ return docID;+ }++ @Override+ public int advance(int target) {+ throw new UnsupportedOperationException("use nextDoc instead");+ }++ @Override+ public boolean advanceExact(int target) throws IOException {+ // needed in IndexSorter#StringSorter+ docID = target;+ ordUpto = 0;+ return ords[docID] != null;+ }++ @Override+ public long nextOrd() {+ if (ordUpto == ords[docID].length) {+ return NO_MORE_ORDS;+ } else {+ return ords[docID][ordUpto++];+ }+ }++ @Override+ public long cost() {+ return in.cost();+ }++ @Override+ public BytesRef lookupOrd(long ord) throws IOException {+ return in.lookupOrd(ord);+ }++ @Override+ public long getValueCount() {+ return in.getValueCount();+ }+ }}diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java b/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.javanew file mode 100644index 00000000000..cc7739c3407--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java@@ -0,0 +1,513 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.lucene.index;++import java.io.IOException;+import java.util.Arrays;+import java.util.HashMap;+import java.util.Iterator;+import java.util.Map;++import org.apache.lucene.codecs.DocValuesProducer;+import org.apache.lucene.codecs.FieldsProducer;+import org.apache.lucene.codecs.NormsProducer;+import org.apache.lucene.codecs.PointsReader;+import org.apache.lucene.codecs.StoredFieldsReader;+import org.apache.lucene.codecs.TermVectorsReader;+import org.apache.lucene.search.Sort;+import org.apache.lucene.util.Bits;+import org.apache.lucene.util.FixedBitSet;++import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;++/**+ * An {@link org.apache.lucene.index.CodecReader} which supports sorting documents by a given+ * {@link Sort}. This can be used to re-sort and index after it's been created by wrapping all+ * readers of the index with this reader and adding it to a fresh IndexWriter via+ * {@link IndexWriter#addIndexes(CodecReader...)}.+ *+ * @lucene.experimental+ */+public final class SortingCodecReader extends FilterCodecReader {++ private final Map<String, NumericDocValuesWriter.CachedNumericDVs> cachedNumericDVs = new HashMap<>();++ private final Map<String, BinaryDocValuesWriter.CachedBinaryDVs> cachedBinaryDVs = new HashMap<>();++ private final Map<String, int[]> cachedSortedDVs = new HashMap<>();++ // TODO: pack long[][] into an int[] (offset) and long[] instead:+ private final Map<String, long[][]> cachedSortedSetDVs = new HashMap<>();++ private final Map<String, long[][]> cachedSortedNumericDVs = new HashMap<>();++ private static class SortingBits implements Bits {++ private final Bits in;+ private final Sorter.DocMap docMap;++ SortingBits(final Bits in, Sorter.DocMap docMap) {+ this.in = in;+ this.docMap = docMap;+ }++ @Override+ public boolean get(int index) {+ return in.get(docMap.newToOld(index));+ }++ @Override+ public int length() {+ return in.length();+ }+ }++ private static class SortingPointValues extends PointValues {++ private final PointValues in;+ private final Sorter.DocMap docMap;++ SortingPointValues(final PointValues in, Sorter.DocMap docMap) {+ this.in = in;+ this.docMap = docMap;+ }++ @Override+ public void intersect(IntersectVisitor visitor) throws IOException {+ in.intersect(new IntersectVisitor() {+ @Override+ public void visit(int docID) throws IOException {+ visitor.visit(docMap.oldToNew(docID));+ }++ @Override+ public void visit(int docID, byte[] packedValue) throws IOException {+ visitor.visit(docMap.oldToNew(docID), packedValue);+ }++ @Override+ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {+ return visitor.compare(minPackedValue, maxPackedValue);+ }+ });+ }++ @Override+ public long estimatePointCount(IntersectVisitor visitor) {+ return in.estimatePointCount(visitor);+ }++ @Override+ public byte[] getMinPackedValue() throws IOException {+ return in.getMinPackedValue();+ }++ @Override+ public byte[] getMaxPackedValue() throws IOException {+ return in.getMaxPackedValue();+ }++ @Override+ public int getNumDimensions() throws IOException {+ return in.getNumDimensions();+ }++ @Override+ public int getNumIndexDimensions() throws IOException {+ return in.getNumIndexDimensions();+ }++ @Override+ public int getBytesPerDimension() throws IOException {+ return in.getBytesPerDimension();+ }++ @Override+ public long size() {+ return in.size();+ }++ @Override+ public int getDocCount() {+ return in.getDocCount();+ }+ }++++++ /** Return a sorted view of <code>reader</code> according to the order+ * defined by <code>sort</code>. If the reader is already sorted, this+ * method might return the reader as-is. */+ public static CodecReader wrap(CodecReader reader, Sort sort) throws IOException {+ return wrap(reader, new Sorter(sort).sort(reader), sort);+ }++ /** Expert: same as {@link #wrap(org.apache.lucene.index.CodecReader, Sort)} but operates directly on a {@link Sorter.DocMap}. */+ static CodecReader wrap(CodecReader reader, Sorter.DocMap docMap, Sort sort) {+ LeafMetaData metaData = reader.getMetaData();+ LeafMetaData newMetaData = new LeafMetaData(metaData.getCreatedVersionMajor(), metaData.getMinVersion(), sort);+ if (docMap == null) {+ // the reader is already sorted+ return new FilterCodecReader(reader) {+ @Override+ public CacheHelper getCoreCacheHelper() {+ return null;+ }++ @Override+ public CacheHelper getReaderCacheHelper() {+ return null;+ }++ @Override+ public LeafMetaData getMetaData() {+ return newMetaData;+ }++ @Override+ public String toString() {+ return "SortingCodecReader(" + in + ")";+ }+ };+ }+ if (reader.maxDoc() != docMap.size()) {+ throw new IllegalArgumentException("reader.maxDoc() should be equal to docMap.size(), got" + reader.maxDoc() + " != " + docMap.size());+ }+ assert Sorter.isConsistent(docMap);+ return new SortingCodecReader(reader, docMap, newMetaData);+ }++ final Sorter.DocMap docMap; // pkg-protected to avoid synthetic accessor methods+ final LeafMetaData metaData;++ private SortingCodecReader(final CodecReader in, final Sorter.DocMap docMap, LeafMetaData metaData) {+ super(in);+ this.docMap = docMap;+ this.metaData = metaData;+ }+++ @Override+ public FieldsProducer getPostingsReader() {+ FieldsProducer postingsReader = in.getPostingsReader();+ return new FieldsProducer() {+ @Override+ public void close() throws IOException {+ postingsReader.close();+ }++ @Override+ public void checkIntegrity() throws IOException {+ postingsReader.checkIntegrity();+ }++ @Override+ public Iterator<String> iterator() {+ return postingsReader.iterator();+ }++ @Override+ public Terms terms(String field) throws IOException {+ Terms terms = postingsReader.terms(field);+ return terms == null ? null : new FreqProxTermsWriter.SortingTerms(terms,+ in.getFieldInfos().fieldInfo(field).getIndexOptions(), docMap);+ }++ @Override+ public int size() {+ return postingsReader.size();+ }++ @Override+ public long ramBytesUsed() {+ return postingsReader.ramBytesUsed();+ }+ };+ }++ @Override+ public StoredFieldsReader getFieldsReader() {+ StoredFieldsReader delegate = in.getFieldsReader();+ return newStoredFieldsReader(delegate);+ }++ private StoredFieldsReader newStoredFieldsReader(StoredFieldsReader delegate) {+ return new StoredFieldsReader() {+ @Override+ public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {+ delegate.visitDocument(docMap.newToOld(docID), visitor);+ }++ @Override+ public StoredFieldsReader clone() {+ return newStoredFieldsReader(delegate.clone());+ }++ @Override+ public void checkIntegrity() throws IOException {+ delegate.checkIntegrity();+ }++ @Override+ public void close() throws IOException {+ delegate.close();+ }++ @Override+ public long ramBytesUsed() {+ return delegate.ramBytesUsed();+ }+ };+ }++ @Override+ public Bits getLiveDocs() {+ final Bits inLiveDocs = in.getLiveDocs();+ if (inLiveDocs == null) {+ return null;+ } else {+ return new SortingBits(inLiveDocs, docMap);+ }+ }++ @Override+ public PointsReader getPointsReader() {+ final PointsReader delegate = in.getPointsReader();+ return new PointsReader() {+ @Override+ public void checkIntegrity() throws IOException {+ delegate.checkIntegrity();+ }++ @Override+ public PointValues getValues(String field) throws IOException {+ return new SortingPointValues(delegate.getValues(field), docMap);+ }++ @Override+ public void close() throws IOException {+ delegate.close();+ }++ @Override+ public long ramBytesUsed() {+ return delegate.ramBytesUsed();+ }+ };+ }++ private final Map<String, NumericDocValuesWriter.CachedNumericDVs> cachedNorms = new HashMap<>();++ @Override+ public NormsProducer getNormsReader() {+ final NormsProducer delegate = in.getNormsReader();+ return new NormsProducer() {+ @Override+ public NumericDocValues getNorms(FieldInfo field) throws IOException {+ return produceNumericDocValues(field, delegate.getNorms(field), cachedNorms);+ }++ @Override+ public void checkIntegrity() throws IOException {+ delegate.checkIntegrity();+ }++ @Override+ public void close() throws IOException {+ delegate.close();+ }++ @Override+ public long ramBytesUsed() {+ return delegate.ramBytesUsed();+ }+ };+ }++ private NumericDocValues produceNumericDocValues(FieldInfo field, NumericDocValues oldNorms,+ Map<String, NumericDocValuesWriter.CachedNumericDVs> cachedNorms) throws IOException {+ NumericDocValuesWriter.CachedNumericDVs norms;+ synchronized (cachedNorms) {+ norms = cachedNorms.get(field);+ if (norms == null) {+ FixedBitSet docsWithField = new FixedBitSet(maxDoc());+ long[] values = new long[maxDoc()];+ while (true) {+ int docID = oldNorms.nextDoc();+ if (docID == NO_MORE_DOCS) {+ break;+ }+ int newDocID = docMap.oldToNew(docID);+ docsWithField.set(newDocID);+ values[newDocID] = oldNorms.longValue();+ }+ norms = new NumericDocValuesWriter.CachedNumericDVs(values, docsWithField);+ cachedNorms.put(field.name, norms);+ }+ }+ return new NumericDocValuesWriter.SortingNumericDocValues(norms);+ }++ @Override+ public DocValuesProducer getDocValuesReader() {+ final DocValuesProducer delegate = in.getDocValuesReader();+ return new DocValuesProducer() {+ @Override+ public NumericDocValues getNumeric(FieldInfo field) throws IOException {+ return produceNumericDocValues(field,delegate.getNumeric(field), cachedNumericDVs);+ }++ @Override+ public BinaryDocValues getBinary(FieldInfo field) throws IOException {+ final BinaryDocValues oldDocValues = delegate.getBinary(field);+ BinaryDocValuesWriter.CachedBinaryDVs dvs;+ synchronized (cachedBinaryDVs) {+ dvs = cachedBinaryDVs.get(field);+ if (dvs == null) {+ dvs = BinaryDocValuesWriter.sortDocValues(maxDoc(), docMap, oldDocValues);+ cachedBinaryDVs.put(field.name, dvs);+ }+ }+ return new BinaryDocValuesWriter.SortingBinaryDocValues(dvs);+ }++ @Override+ public SortedDocValues getSorted(FieldInfo field) throws IOException {+ SortedDocValues oldDocValues = delegate.getSorted(field);+ int[] ords;+ synchronized (cachedSortedDVs) {+ ords = cachedSortedDVs.get(field);+ if (ords == null) {+ ords = new int[maxDoc()];+ Arrays.fill(ords, -1);+ int docID;+ while ((docID = oldDocValues.nextDoc()) != NO_MORE_DOCS) {+ int newDocID = docMap.oldToNew(docID);+ ords[newDocID] = oldDocValues.ordValue();+ }+ cachedSortedDVs.put(field.name, ords);+ }+ }++ return new SortedDocValuesWriter.SortingSortedDocValues(oldDocValues, ords);+ }++ @Override+ public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {+ final SortedNumericDocValues oldDocValues = delegate.getSortedNumeric(field);+ long[][] values;+ synchronized (cachedSortedNumericDVs) {+ values = cachedSortedNumericDVs.get(field);+ if (values == null) {+ values = SortedNumericDocValuesWriter.sortDocValues(maxDoc(), docMap, oldDocValues);+ cachedSortedNumericDVs.put(field.name, values);+ }+ }++ return new SortedNumericDocValuesWriter.SortingSortedNumericDocValues(oldDocValues, values);+ }++ @Override+ public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {+ SortedSetDocValues oldDocValues = delegate.getSortedSet(field);+ long[][] ords;+ synchronized (cachedSortedSetDVs) {+ ords = cachedSortedSetDVs.get(field);+ if (ords == null) {+ ords = SortedSetDocValuesWriter.sortDocValues(maxDoc(), docMap, oldDocValues);+ cachedSortedSetDVs.put(field.name, ords);+ }+ }+ return new SortedSetDocValuesWriter.SortingSortedSetDocValues(oldDocValues, ords);+ }++ @Override+ public void checkIntegrity() throws IOException {+ delegate.checkIntegrity();+ }++ @Override+ public void close() throws IOException {+ delegate.close();+ }++ @Override+ public long ramBytesUsed() {+ return delegate.ramBytesUsed();+ }+ };+ }++ @Override+ public TermVectorsReader getTermVectorsReader() {+ return newTermVectorsReader(in.getTermVectorsReader());+ }++ private TermVectorsReader newTermVectorsReader(TermVectorsReader delegate) {+ return new TermVectorsReader() {+ @Override+ public Fields get(int doc) throws IOException {+ return delegate.get(docMap.newToOld(doc));+ }++ @Override+ public void checkIntegrity() throws IOException {+ delegate.checkIntegrity();+ }++ @Override+ public TermVectorsReader clone() {+ return newTermVectorsReader(delegate.clone());+ }++ @Override+ public void close() throws IOException {+ delegate.close();+ }++ @Override+ public long ramBytesUsed() {+ return delegate.ramBytesUsed();+ }+ };+ }++ @Override+ public String toString() {+ return "SortingCodecReader(" + in + ")";+ }++ // no caching on sorted views+ @Override+ public CacheHelper getCoreCacheHelper() {+ return null;+ }++ @Override+ public CacheHelper getReaderCacheHelper() {+ return null;+ }++ @Override+ public LeafMetaData getMetaData() {+ return metaData;+ }++}diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.javadeleted file mode 100644index 7ec56964fb0..00000000000--- a/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java+++ /dev/null@@ -1,1267 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--package org.apache.lucene.index;--import java.io.IOException;-import java.util.Arrays;-import java.util.HashMap;-import java.util.Map;--import org.apache.lucene.search.DocIdSetIterator;-import org.apache.lucene.search.Sort;-import org.apache.lucene.store.ByteBuffersDataInput;-import org.apache.lucene.store.ByteBuffersDataOutput;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.util.ArrayUtil;-import org.apache.lucene.util.BitSet;-import org.apache.lucene.util.Bits;-import org.apache.lucene.util.BytesRef;-import org.apache.lucene.util.FixedBitSet;-import org.apache.lucene.util.TimSorter;-import org.apache.lucene.util.automaton.CompiledAutomaton;--import static org.apache.lucene.index.SortedSetDocValues.NO_MORE_ORDS;-import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;--/**- * An {@link org.apache.lucene.index.LeafReader} which supports sorting documents by a given- * {@link Sort}. This is package private and is only used by Lucene for BWC when it needs to merge- * an unsorted flushed segment built by an older version (newly flushed segments are sorted since version 7.0).- *- * @lucene.experimental- */-class SortingLeafReader extends FilterLeafReader {-- //TODO remove from here; move to FreqProxTermsWriter or FreqProxFields?- static class SortingFields extends FilterFields {-- private final Sorter.DocMap docMap;- private final FieldInfos infos;-- public SortingFields(final Fields in, FieldInfos infos, Sorter.DocMap docMap) {- super(in);- this.docMap = docMap;- this.infos = infos;- }-- @Override- public Terms terms(final String field) throws IOException {- Terms terms = in.terms(field);- if (terms == null) {- return null;- } else {- return new SortingTerms(terms, infos.fieldInfo(field).getIndexOptions(), docMap);- }- }-- }-- private static class SortingTerms extends FilterTerms {-- private final Sorter.DocMap docMap;- private final IndexOptions indexOptions;-- public SortingTerms(final Terms in, IndexOptions indexOptions, final Sorter.DocMap docMap) {- super(in);- this.docMap = docMap;- this.indexOptions = indexOptions;- }-- @Override- public TermsEnum iterator() throws IOException {- return new SortingTermsEnum(in.iterator(), docMap, indexOptions, hasPositions());- }-- @Override- public TermsEnum intersect(CompiledAutomaton compiled, BytesRef startTerm)- throws IOException {- return new SortingTermsEnum(in.intersect(compiled, startTerm), docMap, indexOptions, hasPositions());- }-- }-- private static class SortingTermsEnum extends FilterTermsEnum {-- final Sorter.DocMap docMap; // pkg-protected to avoid synthetic accessor methods- private final IndexOptions indexOptions;- private final boolean hasPositions;-- public SortingTermsEnum(final TermsEnum in, Sorter.DocMap docMap, IndexOptions indexOptions, boolean hasPositions) {- super(in);- this.docMap = docMap;- this.indexOptions = indexOptions;- this.hasPositions = hasPositions;- }-- @Override- public PostingsEnum postings( PostingsEnum reuse, final int flags) throws IOException {-- if (hasPositions && PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)) {- final PostingsEnum inReuse;- final SortingPostingsEnum wrapReuse;- if (reuse != null && reuse instanceof SortingPostingsEnum) {- // if we're asked to reuse the given DocsEnum and it is Sorting, return- // the wrapped one, since some Codecs expect it.- wrapReuse = (SortingPostingsEnum) reuse;- inReuse = wrapReuse.getWrapped();- } else {- wrapReuse = null;- inReuse = reuse;- }-- final PostingsEnum inDocsAndPositions = in.postings(inReuse, flags);- // we ignore the fact that offsets may be stored but not asked for,- // since this code is expected to be used during addIndexes which will- // ask for everything. if that assumption changes in the future, we can- // factor in whether 'flags' says offsets are not required.- final boolean storeOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;- return new SortingPostingsEnum(docMap.size(), wrapReuse, inDocsAndPositions, docMap, storeOffsets);- }-- final PostingsEnum inReuse;- final SortingDocsEnum wrapReuse;- if (reuse != null && reuse instanceof SortingDocsEnum) {- // if we're asked to reuse the given DocsEnum and it is Sorting, return- // the wrapped one, since some Codecs expect it.- wrapReuse = (SortingDocsEnum) reuse;- inReuse = wrapReuse.getWrapped();- } else {- wrapReuse = null;- inReuse = reuse;- }-- final PostingsEnum inDocs = in.postings(inReuse, flags);- final boolean withFreqs = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >=0 && PostingsEnum.featureRequested(flags, PostingsEnum.FREQS);- return new SortingDocsEnum(docMap.size(), wrapReuse, inDocs, withFreqs, docMap);- }-- }-- static class SortingBinaryDocValues extends BinaryDocValues {-- private final CachedBinaryDVs dvs;- private int docID = -1;-- public SortingBinaryDocValues(CachedBinaryDVs dvs) {- this.dvs = dvs;- }-- @Override- public int nextDoc() {- if (docID+1 == dvs.docsWithField.length()) {- docID = NO_MORE_DOCS;- } else {- docID = dvs.docsWithField.nextSetBit(docID+1);- }-- return docID;- }-- @Override- public int docID() {- return docID;- }-- @Override- public int advance(int target) {- docID = dvs.docsWithField.nextSetBit(target);- return docID;- }-- @Override- public boolean advanceExact(int target) throws IOException {- docID = target;- return dvs.docsWithField.get(target);- }-- @Override- public BytesRef binaryValue() {- return dvs.values[docID];- }-- @Override- public long cost() {- return dvs.docsWithField.cardinality();- }- }-- private final Map<String,CachedNumericDVs> cachedNumericDVs = new HashMap<>();-- static class CachedNumericDVs {- private final long[] values;- private final BitSet docsWithField;-- public CachedNumericDVs(long[] values, BitSet docsWithField) {- this.values = values;- this.docsWithField = docsWithField;- }- }-- private final Map<String,CachedBinaryDVs> cachedBinaryDVs = new HashMap<>();-- static class CachedBinaryDVs {- // TODO: at least cutover to BytesRefArray here:- private final BytesRef[] values;- private final BitSet docsWithField;-- public CachedBinaryDVs(BytesRef[] values, BitSet docsWithField) {- this.values = values;- this.docsWithField = docsWithField;- }- }-- private final Map<String,int[]> cachedSortedDVs = new HashMap<>();-- static class SortingNumericDocValues extends NumericDocValues {-- private final CachedNumericDVs dvs;- private int docID = -1;-- public SortingNumericDocValues(CachedNumericDVs dvs) {- this.dvs = dvs;- }-- @Override- public int docID() {- return docID;- }-- @Override- public int nextDoc() {- if (docID+1 == dvs.docsWithField.length()) {- docID = NO_MORE_DOCS;- } else {- docID = dvs.docsWithField.nextSetBit(docID+1);- }-- return docID;- }-- @Override- public int advance(int target) {- docID = dvs.docsWithField.nextSetBit(target);- return docID;- }-- @Override- public boolean advanceExact(int target) throws IOException {- docID = target;- return dvs.docsWithField.get(target);- }-- @Override- public long longValue() {- return dvs.values[docID];- }-- @Override- public long cost() {- return dvs.docsWithField.cardinality();- }- }-- private static class SortingBits implements Bits {-- private final Bits in;- private final Sorter.DocMap docMap;-- public SortingBits(final Bits in, Sorter.DocMap docMap) {- this.in = in;- this.docMap = docMap;- }-- @Override- public boolean get(int index) {- return in.get(docMap.newToOld(index));- }-- @Override- public int length() {- return in.length();- }- }-- private static class SortingPointValues extends PointValues {-- private final PointValues in;- private final Sorter.DocMap docMap;-- public SortingPointValues(final PointValues in, Sorter.DocMap docMap) {- this.in = in;- this.docMap = docMap;- }-- @Override- public void intersect(IntersectVisitor visitor) throws IOException {- in.intersect(new IntersectVisitor() {- @Override- public void visit(int docID) throws IOException {- visitor.visit(docMap.oldToNew(docID));- }-- @Override- public void visit(int docID, byte[] packedValue) throws IOException {- visitor.visit(docMap.oldToNew(docID), packedValue);- }-- @Override- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {- return visitor.compare(minPackedValue, maxPackedValue);- }- });- }-- @Override- public long estimatePointCount(IntersectVisitor visitor) {- return in.estimatePointCount(visitor);- }-- @Override- public byte[] getMinPackedValue() throws IOException {- return in.getMinPackedValue();- }-- @Override- public byte[] getMaxPackedValue() throws IOException {- return in.getMaxPackedValue();- }-- @Override- public int getNumDimensions() throws IOException {- return in.getNumDimensions();- }-- @Override- public int getNumIndexDimensions() throws IOException {- return in.getNumIndexDimensions();- }-- @Override- public int getBytesPerDimension() throws IOException {- return in.getBytesPerDimension();- }-- @Override- public long size() {- return in.size();- }-- @Override- public int getDocCount() {- return in.getDocCount();- }- }-- static class SortingSortedDocValues extends SortedDocValues {-- private final SortedDocValues in;- private final int[] ords;- private int docID = -1;-- SortingSortedDocValues(SortedDocValues in, int[] ords) {- this.in = in;- this.ords = ords;- assert ords != null;- }-- @Override- public int docID() {- return docID;- }-- @Override- public int nextDoc() {- while (true) {- docID++;- if (docID == ords.length) {- docID = NO_MORE_DOCS;- break;- }- if (ords[docID] != -1) {- break;- }- // skip missing docs- }-- return docID;- }-- @Override- public int advance(int target) {- if (target >= ords.length) {- docID = NO_MORE_DOCS;- } else {- docID = target;- if (ords[docID] == -1) {- nextDoc();- }- }- return docID;- }-- @Override- public boolean advanceExact(int target) throws IOException {- docID = target;- return ords[target] != -1;- }-- @Override- public int ordValue() {- return ords[docID];- }-- @Override- public long cost() {- return in.cost();- }-- @Override- public BytesRef lookupOrd(int ord) throws IOException {- return in.lookupOrd(ord);- }-- @Override- public int getValueCount() {- return in.getValueCount();- }- }-- // TODO: pack long[][] into an int[] (offset) and long[] instead:- private final Map<String,long[][]> cachedSortedSetDVs = new HashMap<>();-- static class SortingSortedSetDocValues extends SortedSetDocValues {-- private final SortedSetDocValues in;- private final long[][] ords;- private int docID = -1;- private int ordUpto;-- SortingSortedSetDocValues(SortedSetDocValues in, long[][] ords) {- this.in = in;- this.ords = ords;- }-- @Override- public int docID() {- return docID;- }-- @Override- public int nextDoc() {- while (true) {- docID++;- if (docID == ords.length) {- docID = NO_MORE_DOCS;- break;- }- if (ords[docID] != null) {- break;- }- // skip missing docs- }- ordUpto = 0;- return docID;- }-- @Override- public int advance(int target) {- if (target >= ords.length) {- docID = NO_MORE_DOCS;- } else {- docID = target;- if (ords[docID] == null) {- nextDoc();- } else {- ordUpto = 0;- }- }- return docID;- }-- @Override- public boolean advanceExact(int target) throws IOException {- docID = target;- ordUpto = 0;- return ords[docID] != null;- }-- @Override- public long nextOrd() {- if (ordUpto == ords[docID].length) {- return NO_MORE_ORDS;- } else {- return ords[docID][ordUpto++];- }- }-- @Override- public long cost() {- return in.cost();- }-- @Override- public BytesRef lookupOrd(long ord) throws IOException {- return in.lookupOrd(ord);- }-- @Override- public long getValueCount() {- return in.getValueCount();- }- }-- private final Map<String,long[][]> cachedSortedNumericDVs = new HashMap<>();-- static class SortingSortedNumericDocValues extends SortedNumericDocValues {- private final SortedNumericDocValues in;- private final long[][] values;- private int docID = -1;- private int upto;-- SortingSortedNumericDocValues(SortedNumericDocValues in, long[][] values) {- this.in = in;- this.values = values;- }-- @Override- public int docID() {- return docID;- }-- @Override- public int nextDoc() {- while (true) {- docID++;- if (docID == values.length) {- docID = NO_MORE_DOCS;- break;- }- if (values[docID] != null) {- break;- }- // skip missing docs- }- upto = 0;- return docID;- }-- @Override- public int advance(int target) {- if (target >= values.length) {- docID = NO_MORE_DOCS;- return docID;- } else {- docID = target-1;- return nextDoc();- }- }-- @Override- public boolean advanceExact(int target) throws IOException {- docID = target;- upto = 0;- return values[docID] != null;- }-- @Override- public long nextValue() {- if (upto == values[docID].length) {- throw new AssertionError();- } else {- return values[docID][upto++];- }- }-- @Override- public long cost() {- return in.cost();- }-- @Override- public int docValueCount() {- return values[docID].length;- }- }-- static class SortingDocsEnum extends FilterPostingsEnum {-- private static final class DocFreqSorter extends TimSorter {-- private int[] docs;- private int[] freqs;- private final int[] tmpDocs;- private int[] tmpFreqs;-- public DocFreqSorter(int maxDoc) {- super(maxDoc / 64);- this.tmpDocs = new int[maxDoc / 64];- }-- public void reset(int[] docs, int[] freqs) {- this.docs = docs;- this.freqs = freqs;- if (freqs != null && tmpFreqs == null) {- tmpFreqs = new int[tmpDocs.length];- }- }-- @Override- protected int compare(int i, int j) {- return docs[i] - docs[j];- }-- @Override- protected void swap(int i, int j) {- int tmpDoc = docs[i];- docs[i] = docs[j];- docs[j] = tmpDoc;-- if (freqs != null) {- int tmpFreq = freqs[i];- freqs[i] = freqs[j];- freqs[j] = tmpFreq;- }- }-- @Override- protected void copy(int src, int dest) {- docs[dest] = docs[src];- if (freqs != null) {- freqs[dest] = freqs[src];- }- }-- @Override- protected void save(int i, int len) {- System.arraycopy(docs, i, tmpDocs, 0, len);- if (freqs != null) {- System.arraycopy(freqs, i, tmpFreqs, 0, len);- }- }-- @Override- protected void restore(int i, int j) {- docs[j] = tmpDocs[i];- if (freqs != null) {- freqs[j] = tmpFreqs[i];- }- }-- @Override- protected int compareSaved(int i, int j) {- return tmpDocs[i] - docs[j];- }- }-- private final int maxDoc;- private final DocFreqSorter sorter;- private int[] docs;- private int[] freqs;- private int docIt = -1;- private final int upto;- private final boolean withFreqs;-- SortingDocsEnum(int maxDoc, SortingDocsEnum reuse, final PostingsEnum in, boolean withFreqs, final Sorter.DocMap docMap) throws IOException {- super(in);- this.maxDoc = maxDoc;- this.withFreqs = withFreqs;- if (reuse != null) {- if (reuse.maxDoc == maxDoc) {- sorter = reuse.sorter;- } else {- sorter = new DocFreqSorter(maxDoc);- }- docs = reuse.docs;- freqs = reuse.freqs; // maybe null- } else {- docs = new int[64];- sorter = new DocFreqSorter(maxDoc);- }- docIt = -1;- int i = 0;- int doc;- if (withFreqs) {- if (freqs == null || freqs.length < docs.length) {- freqs = new int[docs.length];- }- while ((doc = in.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS){- if (i >= docs.length) {- docs = ArrayUtil.grow(docs, docs.length + 1);- freqs = ArrayUtil.grow(freqs, freqs.length + 1);- }- docs[i] = docMap.oldToNew(doc);- freqs[i] = in.freq();- ++i;- }- } else {- freqs = null;- while ((doc = in.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS){- if (i >= docs.length) {- docs = ArrayUtil.grow(docs, docs.length + 1);- }- docs[i++] = docMap.oldToNew(doc);- }- }- // TimSort can save much time compared to other sorts in case of- // reverse sorting, or when sorting a concatenation of sorted readers- sorter.reset(docs, freqs);- sorter.sort(0, i);- upto = i;- }-- // for testing- boolean reused(PostingsEnum other) {- if (other == null || !(other instanceof SortingDocsEnum)) {- return false;- }- return docs == ((SortingDocsEnum) other).docs;- }-- @Override- public int advance(final int target) throws IOException {- // need to support it for checkIndex, but in practice it won't be called, so- // don't bother to implement efficiently for now.- return slowAdvance(target);- }-- @Override- public int docID() {- return docIt < 0 ? -1 : docIt >= upto ? NO_MORE_DOCS : docs[docIt];- }-- @Override- public int freq() throws IOException {- return withFreqs && docIt < upto ? freqs[docIt] : 1;- }-- @Override- public int nextDoc() throws IOException {- if (++docIt >= upto) return NO_MORE_DOCS;- return docs[docIt];- }-- /** Returns the wrapped {@link PostingsEnum}. */- PostingsEnum getWrapped() {- return in;- }-- // we buffer up docs/freqs only, don't forward any positions requests to underlying enum-- @Override- public int nextPosition() throws IOException {- return -1;- }-- @Override- public int startOffset() throws IOException {- return -1;- }-- @Override- public int endOffset() throws IOException {- return -1;- }-- @Override- public BytesRef getPayload() throws IOException {- return null;- }- }-- static class SortingPostingsEnum extends FilterPostingsEnum {-- /**- * A {@link TimSorter} which sorts two parallel arrays of doc IDs and- * offsets in one go. Everytime a doc ID is 'swapped', its corresponding offset- * is swapped too.- */- private static final class DocOffsetSorter extends TimSorter {-- private int[] docs;- private long[] offsets;- private final int[] tmpDocs;- private final long[] tmpOffsets;-- public DocOffsetSorter(int maxDoc) {- super(maxDoc / 64);- this.tmpDocs = new int[maxDoc / 64];- this.tmpOffsets = new long[maxDoc / 64];- }-- public void reset(int[] docs, long[] offsets) {- this.docs = docs;- this.offsets = offsets;- }-- @Override- protected int compare(int i, int j) {- return docs[i] - docs[j];- }-- @Override- protected void swap(int i, int j) {- int tmpDoc = docs[i];- docs[i] = docs[j];- docs[j] = tmpDoc;-- long tmpOffset = offsets[i];- offsets[i] = offsets[j];- offsets[j] = tmpOffset;- }-- @Override- protected void copy(int src, int dest) {- docs[dest] = docs[src];- offsets[dest] = offsets[src];- }-- @Override- protected void save(int i, int len) {- System.arraycopy(docs, i, tmpDocs, 0, len);- System.arraycopy(offsets, i, tmpOffsets, 0, len);- }-- @Override- protected void restore(int i, int j) {- docs[j] = tmpDocs[i];- offsets[j] = tmpOffsets[i];- }-- @Override- protected int compareSaved(int i, int j) {- return tmpDocs[i] - docs[j];- }- }-- private final int maxDoc;- private final DocOffsetSorter sorter;- private int[] docs;- private long[] offsets;- private final int upto;-- private final ByteBuffersDataInput postingInput;- private final boolean storeOffsets;-- private int docIt = -1;- private int pos;- private int startOffset = -1;- private int endOffset = -1;- private final BytesRef payload;- private int currFreq;-- private final ByteBuffersDataOutput buffer;-- SortingPostingsEnum(int maxDoc, SortingPostingsEnum reuse, final PostingsEnum in, Sorter.DocMap docMap, boolean storeOffsets) throws IOException {- super(in);- this.maxDoc = maxDoc;- this.storeOffsets = storeOffsets;- if (reuse != null) {- docs = reuse.docs;- offsets = reuse.offsets;- payload = reuse.payload;- buffer = reuse.buffer;- buffer.reset();- if (reuse.maxDoc == maxDoc) {- sorter = reuse.sorter;- } else {- sorter = new DocOffsetSorter(maxDoc);- }- } else {- docs = new int[32];- offsets = new long[32];- payload = new BytesRef(32);- buffer = ByteBuffersDataOutput.newResettableInstance();- sorter = new DocOffsetSorter(maxDoc);- }-- int doc;- int i = 0;- while ((doc = in.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {- if (i == docs.length) {- final int newLength = ArrayUtil.oversize(i + 1, 4);- docs = ArrayUtil.growExact(docs, newLength);- offsets = ArrayUtil.growExact(offsets, newLength);- }- docs[i] = docMap.oldToNew(doc);- offsets[i] = buffer.size();- addPositions(in, buffer);- i++;- }- upto = i;- sorter.reset(docs, offsets);- sorter.sort(0, upto);-- this.postingInput = buffer.toDataInput();- }-- // for testing- boolean reused(PostingsEnum other) {- if (other == null || !(other instanceof SortingPostingsEnum)) {- return false;- }- return docs == ((SortingPostingsEnum) other).docs;- }-- private void addPositions(final PostingsEnum in, final DataOutput out) throws IOException {- int freq = in.freq();- out.writeVInt(freq);- int previousPosition = 0;- int previousEndOffset = 0;- for (int i = 0; i < freq; i++) {- final int pos = in.nextPosition();- final BytesRef payload = in.getPayload();- // The low-order bit of token is set only if there is a payload, the- // previous bits are the delta-encoded position.- final int token = (pos - previousPosition) << 1 | (payload == null ? 0 : 1);- out.writeVInt(token);- previousPosition = pos;- if (storeOffsets) { // don't encode offsets if they are not stored- final int startOffset = in.startOffset();- final int endOffset = in.endOffset();- out.writeVInt(startOffset - previousEndOffset);- out.writeVInt(endOffset - startOffset);- previousEndOffset = endOffset;- }- if (payload != null) {- out.writeVInt(payload.length);- out.writeBytes(payload.bytes, payload.offset, payload.length);- }- }- }-- @Override- public int advance(final int target) throws IOException {- // need to support it for checkIndex, but in practice it won't be called, so- // don't bother to implement efficiently for now.- return slowAdvance(target);- }-- @Override- public int docID() {- return docIt < 0 ? -1 : docIt >= upto ? NO_MORE_DOCS : docs[docIt];- }-- @Override- public int endOffset() throws IOException {- return endOffset;- }-- @Override- public int freq() throws IOException {- return currFreq;- }-- @Override- public BytesRef getPayload() throws IOException {- return payload.length == 0 ? null : payload;- }-- @Override- public int nextDoc() throws IOException {- if (++docIt >= upto) return DocIdSetIterator.NO_MORE_DOCS;- postingInput.seek(offsets[docIt]);- currFreq = postingInput.readVInt();- // reset variables used in nextPosition- pos = 0;- endOffset = 0;- return docs[docIt];- }-- @Override- public int nextPosition() throws IOException {- final int token = postingInput.readVInt();- pos += token >>> 1;- if (storeOffsets) {- startOffset = endOffset + postingInput.readVInt();- endOffset = startOffset + postingInput.readVInt();- }- if ((token & 1) != 0) {- payload.offset = 0;- payload.length = postingInput.readVInt();- if (payload.length > payload.bytes.length) {- payload.bytes = new byte[ArrayUtil.oversize(payload.length, 1)];- }- postingInput.readBytes(payload.bytes, 0, payload.length);- } else {- payload.length = 0;- }- return pos;- }-- @Override- public int startOffset() throws IOException {- return startOffset;- }-- /** Returns the wrapped {@link PostingsEnum}. */- PostingsEnum getWrapped() {- return in;- }- }-- /** Return a sorted view of <code>reader</code> according to the order- * defined by <code>sort</code>. If the reader is already sorted, this- * method might return the reader as-is. */- public static LeafReader wrap(LeafReader reader, Sort sort) throws IOException {- return wrap(reader, new Sorter(sort).sort(reader));- }-- /** Expert: same as {@link #wrap(org.apache.lucene.index.LeafReader, Sort)} but operates directly on a {@link Sorter.DocMap}. */- static LeafReader wrap(LeafReader reader, Sorter.DocMap docMap) {- if (docMap == null) {- // the reader is already sorted- return reader;- }- if (reader.maxDoc() != docMap.size()) {- throw new IllegalArgumentException("reader.maxDoc() should be equal to docMap.size(), got" + reader.maxDoc() + " != " + docMap.size());- }- assert Sorter.isConsistent(docMap);- return new SortingLeafReader(reader, docMap);- }-- final Sorter.DocMap docMap; // pkg-protected to avoid synthetic accessor methods-- private SortingLeafReader(final LeafReader in, final Sorter.DocMap docMap) {- super(in);- this.docMap = docMap;- }-- @Override- public void document(final int docID, final StoredFieldVisitor visitor) throws IOException {- in.document(docMap.newToOld(docID), visitor);- }-- @Override- public Terms terms(String field) throws IOException {- Terms terms = super.terms(field);- return terms==null ? null : new SortingTerms(terms, in.getFieldInfos().fieldInfo(field).getIndexOptions(), docMap);- }-- @Override- public BinaryDocValues getBinaryDocValues(String field) throws IOException {- final BinaryDocValues oldDocValues = in.getBinaryDocValues(field);- if (oldDocValues == null) return null;- CachedBinaryDVs dvs;- synchronized (cachedBinaryDVs) {- dvs = cachedBinaryDVs.get(field);- if (dvs == null) {- FixedBitSet docsWithField = new FixedBitSet(maxDoc());- BytesRef[] values = new BytesRef[maxDoc()];- while (true) {- int docID = oldDocValues.nextDoc();- if (docID == NO_MORE_DOCS) {- break;- }- int newDocID = docMap.oldToNew(docID);- docsWithField.set(newDocID);- values[newDocID] = BytesRef.deepCopyOf(oldDocValues.binaryValue());- }- dvs = new CachedBinaryDVs(values, docsWithField);- cachedBinaryDVs.put(field, dvs);- }- }- return new SortingBinaryDocValues(dvs);- }--- @Override- public Bits getLiveDocs() {- final Bits inLiveDocs = in.getLiveDocs();- if (inLiveDocs == null) {- return null;- } else {- return new SortingBits(inLiveDocs, docMap);- }- }-- @Override- public PointValues getPointValues(String fieldName) throws IOException {- final PointValues inPointValues = in.getPointValues(fieldName);- if (inPointValues == null) {- return null;- } else {- return new SortingPointValues(inPointValues, docMap);- }- }-- private final Map<String,CachedNumericDVs> cachedNorms = new HashMap<>();-- @Override- public NumericDocValues getNormValues(String field) throws IOException {- final NumericDocValues oldNorms = in.getNormValues(field);- if (oldNorms == null) return null;- CachedNumericDVs norms;- synchronized (cachedNorms) {- norms = cachedNorms.get(field);- if (norms == null) {- FixedBitSet docsWithField = new FixedBitSet(maxDoc());- long[] values = new long[maxDoc()];- while (true) {- int docID = oldNorms.nextDoc();- if (docID == NO_MORE_DOCS) {- break;- }- int newDocID = docMap.oldToNew(docID);- docsWithField.set(newDocID);- values[newDocID] = oldNorms.longValue();- }- norms = new CachedNumericDVs(values, docsWithField);- cachedNorms.put(field, norms);- }- }- return new SortingNumericDocValues(norms);- }-- @Override- public NumericDocValues getNumericDocValues(String field) throws IOException {- final NumericDocValues oldDocValues = in.getNumericDocValues(field);- if (oldDocValues == null) return null;- CachedNumericDVs dvs;- synchronized (cachedNumericDVs) {- dvs = cachedNumericDVs.get(field);- if (dvs == null) {- FixedBitSet docsWithField = new FixedBitSet(maxDoc());- long[] values = new long[maxDoc()];- while (true) {- int docID = oldDocValues.nextDoc();- if (docID == NO_MORE_DOCS) {- break;- }- int newDocID = docMap.oldToNew(docID);- docsWithField.set(newDocID);- values[newDocID] = oldDocValues.longValue();- }- dvs = new CachedNumericDVs(values, docsWithField);- cachedNumericDVs.put(field, dvs);- }- }- return new SortingNumericDocValues(dvs);- }-- @Override- public SortedNumericDocValues getSortedNumericDocValues(String field)- throws IOException {- final SortedNumericDocValues oldDocValues = in.getSortedNumericDocValues(field);- if (oldDocValues == null) {- return null;- }-- long[][] values;- synchronized (cachedSortedNumericDVs) {- values = cachedSortedNumericDVs.get(field);- if (values == null) {- values = new long[maxDoc()][];- int docID;- while ((docID = oldDocValues.nextDoc()) != NO_MORE_DOCS) {- int newDocID = docMap.oldToNew(docID);- long[] docValues = new long[oldDocValues.docValueCount()];- for(int i=0;i<docValues.length;i++) {- docValues[i] = oldDocValues.nextValue();- }- values[newDocID] = docValues;- }- cachedSortedNumericDVs.put(field, values);- }- }-- return new SortingSortedNumericDocValues(oldDocValues, values);- }-- @Override- public SortedDocValues getSortedDocValues(String field) throws IOException {- SortedDocValues oldDocValues = in.getSortedDocValues(field);- if (oldDocValues == null) {- return null;- }-- int[] ords;- synchronized (cachedSortedDVs) {- ords = cachedSortedDVs.get(field);- if (ords == null) {- ords = new int[maxDoc()];- Arrays.fill(ords, -1);- int docID;- while ((docID = oldDocValues.nextDoc()) != NO_MORE_DOCS) {- int newDocID = docMap.oldToNew(docID);- ords[newDocID] = oldDocValues.ordValue();- }- cachedSortedDVs.put(field, ords);- }- }-- return new SortingSortedDocValues(oldDocValues, ords);- }-- @Override- public SortedSetDocValues getSortedSetDocValues(String field) throws IOException {- SortedSetDocValues oldDocValues = in.getSortedSetDocValues(field);- if (oldDocValues == null) {- return null;- }-- long[][] ords;- synchronized (cachedSortedSetDVs) {- ords = cachedSortedSetDVs.get(field);- if (ords == null) {- ords = new long[maxDoc()][];- int docID;- while ((docID = oldDocValues.nextDoc()) != NO_MORE_DOCS) {- int newDocID = docMap.oldToNew(docID);- long[] docOrds = new long[1];- int upto = 0;- while (true) {- long ord = oldDocValues.nextOrd();- if (ord == NO_MORE_ORDS) {- break;- }- if (upto == docOrds.length) {- docOrds = ArrayUtil.grow(docOrds);- }- docOrds[upto++] = ord;- }- ords[newDocID] = ArrayUtil.copyOfSubArray(docOrds, 0, upto);- }- cachedSortedSetDVs.put(field, ords);- }- }-- return new SortingSortedSetDocValues(oldDocValues, ords);- }-- @Override- public Fields getTermVectors(final int docID) throws IOException {- return in.getTermVectors(docMap.newToOld(docID));- }-- @Override- public String toString() {- return "SortingLeafReader(" + in + ")";- }-- // no caching on sorted views-- @Override- public CacheHelper getCoreCacheHelper() {- return null;- }-- @Override- public CacheHelper getReaderCacheHelper() {- return null;- }-}diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSortingCodecReader.java b/lucene/core/src/test/org/apache/lucene/index/TestSortingCodecReader.javanew file mode 100644index 00000000000..c1ac197f571--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/index/TestSortingCodecReader.java@@ -0,0 +1,224 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.lucene.index;++import java.io.IOException;+import java.util.ArrayList;+import java.util.Arrays;+import java.util.Collections;+import java.util.List;+import java.util.Locale;++import com.carrotsearch.randomizedtesting.generators.RandomPicks;+import com.carrotsearch.randomizedtesting.generators.RandomStrings;+import org.apache.lucene.analysis.MockAnalyzer;+import org.apache.lucene.codecs.TermVectorsReader;+import org.apache.lucene.document.BinaryDocValuesField;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.Field;+import org.apache.lucene.document.FieldType;+import org.apache.lucene.document.LongPoint;+import org.apache.lucene.document.NumericDocValuesField;+import org.apache.lucene.document.SortedDocValuesField;+import org.apache.lucene.document.SortedNumericDocValuesField;+import org.apache.lucene.document.SortedSetDocValuesField;+import org.apache.lucene.document.StringField;+import org.apache.lucene.document.TextField;+import org.apache.lucene.search.DocIdSetIterator;+import org.apache.lucene.search.IndexSearcher;+import org.apache.lucene.search.Sort;+import org.apache.lucene.search.SortField;+import org.apache.lucene.search.SortedNumericSortField;+import org.apache.lucene.search.SortedSetSortField;+import org.apache.lucene.search.TermQuery;+import org.apache.lucene.search.TopDocs;+import org.apache.lucene.store.Directory;+import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.IOUtils;+import org.apache.lucene.util.LuceneTestCase;++public class TestSortingCodecReader extends LuceneTestCase {++ public void testSortOnAddIndicesInt() throws IOException {+ Directory tmpDir = newDirectory();+ Directory dir = newDirectory();+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));+ IndexWriter w = new IndexWriter(tmpDir, iwc);+ Document doc = new Document();+ doc.add(new NumericDocValuesField("foo", 18));+ w.addDocument(doc);+++ doc = new Document();+ doc.add(new NumericDocValuesField("foo", -1));+ w.addDocument(doc);+ w.commit();++ doc = new Document();+ doc.add(new NumericDocValuesField("foo", 7));+ w.addDocument(doc);+ w.commit();+ w.close();+ Sort indexSort = new Sort(new SortField("foo", SortField.Type.INT));++ iwc = new IndexWriterConfig(new MockAnalyzer(random())).setIndexSort(indexSort);+ w = new IndexWriter(dir, iwc);+ try (DirectoryReader reader = DirectoryReader.open(tmpDir)) {+ List<CodecReader> readers = new ArrayList<>();+ for (LeafReaderContext ctx : reader.leaves()) {+ CodecReader wrap = SortingCodecReader.wrap(SlowCodecReaderWrapper.wrap(ctx.reader()), indexSort);+ assertTrue(wrap.toString(), wrap.toString().startsWith("SortingCodecReader("));+ readers.add(wrap);++ }+ w.addIndexes(readers.toArray(new CodecReader[0]));+ }+ DirectoryReader r = DirectoryReader.open(w);+ LeafReader leaf = getOnlyLeafReader(r);+ assertEquals(3, leaf.maxDoc());+ NumericDocValues values = leaf.getNumericDocValues("foo");+ assertEquals(0, values.nextDoc());+ assertEquals(-1, values.longValue());+ assertEquals(1, values.nextDoc());+ assertEquals(7, values.longValue());+ assertEquals(2, values.nextDoc());+ assertEquals(18, values.longValue());+ assertNotNull(leaf.getMetaData().getSort());+ IOUtils.close(r, w, dir, tmpDir);+ }++ public void testSortOnAddIndicesRandom() throws IOException {+ try (Directory dir = newDirectory()) {+ int numDocs = atLeast(200);+ int actualNumDocs;+ List<Integer> docIds = new ArrayList<>(numDocs);+ for (int i = 0; i < numDocs; i++) {+ docIds.add(i);+ }+ Collections.shuffle(docIds, random());+ try (RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) {+ for (int i = 0; i < numDocs; i++) {+ int docId = docIds.get(i);+ Document doc = new Document();+ doc.add(new NumericDocValuesField("foo", random().nextInt(20)));+ doc.add(new StringField("id", Integer.toString(docId), Field.Store.YES));+ doc.add(new LongPoint("id", docId));+ doc.add(new TextField("text_field", RandomStrings.randomRealisticUnicodeOfLength(random(), 25), Field.Store.YES));+ doc.add(new SortedNumericDocValuesField("sorted_numeric_dv", docId));+ doc.add(new SortedDocValuesField("binary_sorted_dv", new BytesRef(Integer.toString(docId))));+ doc.add(new BinaryDocValuesField("binary_dv", new BytesRef(Integer.toString(docId))));+ doc.add(new SortedSetDocValuesField("sorted_set_dv", new BytesRef(Integer.toString(docId))));++ FieldType ft = new FieldType(StringField.TYPE_NOT_STORED);+ ft.setStoreTermVectors(true);+ doc.add(new Field("term_vectors", "test" + docId, ft));+ if (rarely() == false) {+ doc.add(new NumericDocValuesField("id", docId));+ doc.add(new SortedSetDocValuesField("sorted_set_sort_field", new BytesRef(String.format(Locale.ROOT, "%06d", docId))));+ doc.add(new SortedDocValuesField("sorted_binary_sort_field", new BytesRef(String.format(Locale.ROOT, "%06d", docId))));+ doc.add(new SortedNumericDocValuesField("sorted_numeric_sort_field", docId));+ } else {+ doc.add(new NumericDocValuesField("alt_id", docId));+ }+ iw.addDocument(doc);+ if (i > 0 && random().nextInt(5) == 0) {+ final int id = RandomPicks.randomFrom(random(), docIds.subList(0, i));+ iw.deleteDocuments(new Term("id", Integer.toString(id)));+ }+ }+ iw.commit();+ actualNumDocs = iw.getDocStats().numDocs;+ }+ Sort indexSort = RandomPicks.randomFrom(random(), Arrays.asList(+ new Sort(new SortField("id", SortField.Type.INT),+ new SortField("alt_id", SortField.Type.INT)),+ new Sort(new SortedSetSortField("sorted_set_sort_field", false),+ new SortField("alt_id", SortField.Type.INT)),+ new Sort(new SortedNumericSortField("sorted_numeric_sort_field", SortField.Type.INT),+ new SortField("alt_id", SortField.Type.INT)),+ new Sort(new SortField("sorted_binary_sort_field", SortField.Type.STRING, false),+ new SortField("alt_id", SortField.Type.INT))+ ));+ try (Directory sortDir = newDirectory()) {+ try (IndexWriter writer = new IndexWriter(sortDir, newIndexWriterConfig().setIndexSort(indexSort))) {+ try (DirectoryReader reader = DirectoryReader.open(dir)) {+ List<CodecReader> readers = new ArrayList<>();+ for (LeafReaderContext ctx : reader.leaves()) {+ CodecReader wrap = SortingCodecReader.wrap(SlowCodecReaderWrapper.wrap(ctx.reader()), indexSort);+ readers.add(wrap);+ TermVectorsReader termVectorsReader = wrap.getTermVectorsReader();+ TermVectorsReader clone = termVectorsReader.clone();+ assertNotSame(termVectorsReader, clone);+ clone.close();+ }+ writer.addIndexes(readers.toArray(new CodecReader[0]));+ }+ assumeTrue("must have at least one doc", actualNumDocs > 0);+ try (DirectoryReader r = DirectoryReader.open(writer)) {+ LeafReader leaf = getOnlyLeafReader(r);+ assertEquals(actualNumDocs, leaf.maxDoc());+ BinaryDocValues binary_dv = leaf.getBinaryDocValues("binary_dv");+ SortedNumericDocValues sorted_numeric_dv = leaf.getSortedNumericDocValues("sorted_numeric_dv");+ SortedSetDocValues sorted_set_dv = leaf.getSortedSetDocValues("sorted_set_dv");+ SortedDocValues binary_sorted_dv = leaf.getSortedDocValues("binary_sorted_dv");+ NumericDocValues ids = leaf.getNumericDocValues("id");+ long prevValue = -1;+ boolean usingAltIds = false;+ for (int i = 0; i < actualNumDocs; i++) {+ int idNext = ids.nextDoc();+ if (idNext == DocIdSetIterator.NO_MORE_DOCS) {+ assertFalse(usingAltIds);+ usingAltIds = true;+ ids = leaf.getNumericDocValues("alt_id");+ idNext = ids.nextDoc();+ binary_dv = leaf.getBinaryDocValues("binary_dv");+ sorted_numeric_dv = leaf.getSortedNumericDocValues("sorted_numeric_dv");+ sorted_set_dv = leaf.getSortedSetDocValues("sorted_set_dv");+ binary_sorted_dv = leaf.getSortedDocValues("binary_sorted_dv");+ prevValue = -1;+ }+ assertTrue(prevValue + " < " + ids.longValue(), prevValue < ids.longValue());+ prevValue = ids.longValue();+ assertTrue(binary_dv.advanceExact(idNext));+ assertTrue(sorted_numeric_dv.advanceExact(idNext));+ assertTrue(sorted_set_dv.advanceExact(idNext));+ assertTrue(binary_sorted_dv.advanceExact(idNext));+ assertEquals(new BytesRef(ids.longValue() + ""), binary_dv.binaryValue());+ assertEquals(new BytesRef(ids.longValue() + ""), binary_sorted_dv.binaryValue());+ assertEquals(new BytesRef(ids.longValue() + ""), sorted_set_dv.lookupOrd(sorted_set_dv.nextOrd()));+ assertEquals(1, sorted_numeric_dv.docValueCount());+ assertEquals(ids.longValue(), sorted_numeric_dv.nextValue());+ Fields termVectors = leaf.getTermVectors(idNext);+ assertTrue(termVectors.terms("term_vectors").iterator().seekExact(new BytesRef("test" + ids.longValue())));+ assertEquals(Long.toString(ids.longValue()), leaf.document(idNext).get("id"));+ IndexSearcher searcher = new IndexSearcher(r);+ TopDocs result = searcher.search(LongPoint.newExactQuery("id", ids.longValue()), 1);+ assertEquals(1, result.totalHits.value);+ assertEquals(idNext, result.scoreDocs[0].doc);++ result = searcher.search(new TermQuery(new Term("id", "" + ids.longValue())), 1);+ assertEquals(1, result.totalHits.value);+ assertEquals(idNext, result.scoreDocs[0].doc);+ }+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, ids.nextDoc());+ }+ }+ }+ }+ }+}