Apache Lucene open-source search software
Allow single-valued updates for SortedNumericDocValues (#16581)
Houston Putman169cbffd12f4
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 16a6728248c..f8a74b323ff 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -407,6 +407,12 @@ New Featuresit is not possible to re-encode to a different quantization width or center vectors again.(Trevor McCulloch)+* GITHUB#16580: Allow in-place updates of SORTED_NUMERIC doc-values via+ IndexWriter#updateSortedNumericDocValue (and IndexWriter#updateDocValues with a single+ SortedNumericDocValuesField). An update sets a single value per matched document; the field may be+ single- or multi-valued. Single-valued columns reuse the numeric update path (including the+ incremental delta overlay), and multi-valued columns are merged per-doc. (Houston Putman)+Improvements---------------------diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValues.java b/lucene/core/src/java/org/apache/lucene/index/DocValues.javaindex c51d29b7d6f..e01dd6e5bfe 100644--- a/lucene/core/src/java/org/apache/lucene/index/DocValues.java+++ b/lucene/core/src/java/org/apache/lucene/index/DocValues.java@@ -199,6 +199,14 @@ public final class DocValues {return new SingletonSortedNumericDocValues(dv);}+ /**+ * Returns whether the provided SortedNumericDocValues is a single-valued view of a+ * NumericDocValues.+ */+ static boolean isSingleton(SortedNumericDocValues dv) {+ return dv instanceof SingletonSortedNumericDocValues;+ }+// some helpers, for transition from fieldcache apis.// as opposed to the LeafReader apis (which must be strict for consistency), these are lenientdiff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.javaindex 9be34e997b2..f84ffd0b817 100644--- a/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java+++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java@@ -153,21 +153,38 @@ abstract class DocValuesUpdate {}}- /** An in-place update to a numeric DocValues field */+ /**+ * An in-place update to a numeric DocValues field.+ *+ * <p>Also used for {@link org.apache.lucene.document.SortedNumericDocValuesField} updates ({@code+ * type == SORTED_NUMERIC}): such an update sets a single value per matched document, so its+ * payload is identical to a numeric update — only the doc-values type differs, which drives the+ * singleton wrapping at write/read time. The updated field may itself be single- or multi-valued:+ * a single-valued column is physically stored as a {@link NumericDocValues} column (the codec+ * only writes an addresses table when a document has more than one value) and reuses the numeric+ * write path, while a multi-valued column is merged per-doc.+ */static final class NumericDocValuesUpdate extends DocValuesUpdate {private final long value;NumericDocValuesUpdate(Term term, String field, long value) {- this(term, field, value, BufferedUpdates.MAX_INT, true);+ this(DocValuesType.NUMERIC, term, field, value, BufferedUpdates.MAX_INT, true);}NumericDocValuesUpdate(Term term, String field, Long value) {- this(term, field, value != null ? value : -1, BufferedUpdates.MAX_INT, value != null);+ this(DocValuesType.NUMERIC, term, field, value);+ }++ /** Creates a numeric or sorted-numeric update, depending on {@code type}. */+ NumericDocValuesUpdate(DocValuesType type, Term term, String field, Long value) {+ this(type, term, field, value != null ? value : -1, BufferedUpdates.MAX_INT, value != null);}private NumericDocValuesUpdate(- Term term, String field, long value, int docIDUpTo, boolean hasValue) {- super(DocValuesType.NUMERIC, term, field, docIDUpTo, hasValue);+ DocValuesType type, Term term, String field, long value, int docIDUpTo, boolean hasValue) {+ super(type, term, field, docIDUpTo, hasValue);+ assert type == DocValuesType.NUMERIC || type == DocValuesType.SORTED_NUMERIC+ : "unexpected type: " + type;this.value = value;}@@ -175,7 +192,7 @@ abstract class DocValuesUpdate {if (docIDUpTo == this.docIDUpTo) {return this;}- return new NumericDocValuesUpdate(term, field, value, docIDUpTo, hasValue);+ return new NumericDocValuesUpdate(type, term, field, value, docIDUpTo, hasValue);}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.javaindex 78e913ce9d1..d79864a62ca 100644--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java@@ -501,6 +501,9 @@ final class DocumentsWriterDeleteQueue implements Accountable, Closeable {for (DocValuesUpdate update : item) {switch (update.type) {case NUMERIC:+ case SORTED_NUMERIC:+ // a sorted-numeric update carries a single value, so it shares the numeric update+ // object and bufferbufferedUpdates.addNumericUpdate((NumericDocValuesUpdate) update, docIDUpto);break;case BINARY:@@ -509,7 +512,6 @@ final class DocumentsWriterDeleteQueue implements Accountable, Closeable {case NONE:case SORTED:case SORTED_SET:- case SORTED_NUMERIC:default:throw new IllegalArgumentException(update.type + " DocValues updates not supported yet!");diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldUpdatesBuffer.java b/lucene/core/src/java/org/apache/lucene/index/FieldUpdatesBuffer.javaindex a4d38dcd203..6408988d06a 100644--- a/lucene/core/src/java/org/apache/lucene/index/FieldUpdatesBuffer.java+++ b/lucene/core/src/java/org/apache/lucene/index/FieldUpdatesBuffer.java@@ -61,11 +61,15 @@ final class FieldUpdatesBuffer {private long maxNumeric = Long.MIN_VALUE;private long minNumeric = Long.MAX_VALUE;private String[] fields;+ // the actual doc-values type of the updated field. A SORTED_NUMERIC update sets a single long+ // value per doc, identical to a numeric update, so it shares the numeric buffering path but keeps+ // its own type so the write/read side can apply the singleton wrapping.+ private final DocValuesType type;private final boolean isNumeric;private boolean finished = false;private FieldUpdatesBuffer(- Counter bytesUsed, DocValuesUpdate initialValue, int docUpTo, boolean isNumeric) {+ Counter bytesUsed, DocValuesUpdate initialValue, int docUpTo, DocValuesType type) {this.bytesUsed = bytesUsed;this.bytesUsed.addAndGet(SELF_SHALLOW_SIZE);termValues = new BytesRefArray(bytesUsed);@@ -77,7 +81,9 @@ final class FieldUpdatesBuffer {hasValues = new FixedBitSet(1);bytesUsed.addAndGet(hasValues.ramBytesUsed());}- this.isNumeric = isNumeric;+ this.type = type;+ // NUMERIC and SORTED_NUMERIC updates both carry a single long, so they share the storage path.+ this.isNumeric = type == DocValuesType.NUMERIC || type == DocValuesType.SORTED_NUMERIC;byteValues = isNumeric ? null : new BytesRefArray(bytesUsed);}@@ -87,7 +93,8 @@ final class FieldUpdatesBuffer {FieldUpdatesBuffer(Counter bytesUsed, DocValuesUpdate.NumericDocValuesUpdate initialValue, int docUpTo) {- this(bytesUsed, initialValue, docUpTo, true);+ // NUMERIC and SORTED_NUMERIC share this buffer; the update carries which one.+ this(bytesUsed, initialValue, docUpTo, initialValue.type);if (initialValue.hasValue()) {numericValues = new long[] {initialValue.getValue()};maxNumeric = minNumeric = initialValue.getValue();@@ -99,12 +106,16 @@ final class FieldUpdatesBuffer {FieldUpdatesBuffer(Counter bytesUsed, DocValuesUpdate.BinaryDocValuesUpdate initialValue, int docUpTo) {- this(bytesUsed, initialValue, docUpTo, false);+ this(bytesUsed, initialValue, docUpTo, DocValuesType.BINARY);if (initialValue.hasValue()) {byteValues.append(initialValue.getValue());}}+ DocValuesType getType() {+ return type;+ }+long getMaxNumeric() {assert isNumeric;if (minNumeric == Long.MAX_VALUE && maxNumeric == Long.MIN_VALUE) {diff --git a/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java b/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.javaindex 13e6db358ca..4ffb200facb 100644--- a/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java+++ b/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java@@ -284,15 +284,21 @@ final class FrozenBufferedUpdates {}if (dvUpdates == null) {if (isNumeric) {+ DocValuesType dvType = value.getType();if (value.hasSingleValue()) {dvUpdates =new NumericDocValuesFieldUpdates.SingleValueNumericDocValuesFieldUpdates(- delGen, updateField, segState.reader.maxDoc(), value.getNumericValue(0));+ delGen,+ updateField,+ dvType,+ segState.reader.maxDoc(),+ value.getNumericValue(0));} else {dvUpdates =new NumericDocValuesFieldUpdates(delGen,updateField,+ dvType,value.getMinNumeric(),value.getMaxNumeric(),segState.reader.maxDoc());diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.javaindex 62204c11d4b..45b59fa46dd 100644--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java@@ -1736,14 +1736,14 @@ public class IndexWriterk -> {switch (update.type) {case NUMERIC:+ case SORTED_NUMERIC:return new NumericDocValuesFieldUpdates(- nextGen, k, rld.info.info.maxDoc());+ nextGen, k, update.type, rld.info.info.maxDoc());case BINARY:return new BinaryDocValuesFieldUpdates(nextGen, k, rld.info.info.maxDoc());case NONE:case SORTED:- case SORTED_NUMERIC:case SORTED_SET:default:throw new AssertionError("type: " + update.type + " is not supported");@@ -1752,6 +1752,7 @@ public class IndexWriterif (update.hasValue()) {switch (update.type) {case NUMERIC:+ case SORTED_NUMERIC:docValuesFieldUpdates.add(leafDocId, ((NumericDocValuesUpdate) update).getValue());break;@@ -1762,7 +1763,6 @@ public class IndexWritercase NONE:case SORTED:case SORTED_SET:- case SORTED_NUMERIC:default:throw new AssertionError("type: " + update.type + " is not supported");}@@ -2017,6 +2017,45 @@ public class IndexWriter}}+ /**+ * Updates a document's {@link org.apache.lucene.index.SortedNumericDocValues} for <code>field+ * </code> to the given <code>value</code>. You can only update fields that already exist in the+ * index, not add new fields through this method. You can only update fields that were indexed+ * only with doc values.+ *+ * <p><b>NOTE:</b> the update sets a single value per matched document, replacing all values that+ * document previously had. The field itself may be single- or multi-valued; documents not matched+ * by <code>term</code> keep their existing (possibly multiple) values.+ *+ * @param term the term to identify the document(s) to be updated+ * @param field field name of the {@link org.apache.lucene.index.SortedNumericDocValues} field+ * @param value new value for the field+ * @return The <a href="#sequence_number">sequence number</a> for this operation+ * @throws CorruptIndexException if the index is corrupt+ * @throws IOException if there is a low-level IO error+ */+ public long updateSortedNumericDocValue(Term term, String field, long value) throws IOException {+ ensureOpen();+ // Checked before the doc-values-type check below, mirroring updateNumericDocValue/+ // updateBinaryDocValue, so an index-sort field surfaces this clear message.+ if (config.getIndexSortFields().contains(field)) {+ throw new IllegalArgumentException(+ "cannot update docvalues field involved in the index sort, field="+ + field+ + ", sort="+ + config.getIndexSort());+ }+ globalFieldNumberMap.verifyOrCreateDvOnlyField(field, DocValuesType.SORTED_NUMERIC, true);+ try {+ return maybeProcessEvents(+ docWriter.updateDocValues(+ new NumericDocValuesUpdate(DocValuesType.SORTED_NUMERIC, term, field, value)));+ } catch (Error tragedy) {+ tragicEvent(tragedy, "updateSortedNumericDocValue");+ throw tragedy;+ }+ }+/*** Updates documents' DocValues fields to the given values. Each field update is applied to the* set of documents that are associated with the {@link Term} to the same value. All updates are@@ -2066,19 +2105,22 @@ public class IndexWriterswitch (dvType) {case NUMERIC:+ case SORTED_NUMERIC:Long value = (Long) f.numericValue();- dvUpdates[i] = new NumericDocValuesUpdate(term, f.name(), value);+ dvUpdates[i] = new NumericDocValuesUpdate(dvType, term, f.name(), value);break;case BINARY:dvUpdates[i] = new BinaryDocValuesUpdate(term, f.name(), f.binaryValue());break;case NONE:case SORTED:- case SORTED_NUMERIC:case SORTED_SET:default:throw new IllegalArgumentException(- "can only update NUMERIC or BINARY fields: field=" + f.name() + ", type=" + dvType);+ "can only update NUMERIC, BINARY or SORTED_NUMERIC fields: field="+ + f.name()+ + ", type="+ + dvType);}}return dvUpdates;@@ -4514,9 +4556,6 @@ public class IndexWriterreturn mergedDeletesAndUpdates;}- /** Sentinel for "no doc-values value" (a reset, or a doc that simply has no value). */- private static final Object DV_NO_VALUE = new Object();-/*** Builds the doc-values-update merge carry-over from the source segments' on-disk state plus* their residual (resolved but not-yet-written) pending updates, remapping to merged docIDs.@@ -4561,7 +4600,9 @@ public class IndexWriterFieldInfos baseFieldInfos = baseline.getFieldInfos();for (FieldInfo fi : current.getFieldInfos()) {DocValuesType t = fi.getDocValuesType();- if (t != DocValuesType.NUMERIC && t != DocValuesType.BINARY) {+ if (t != DocValuesType.NUMERIC+ && t != DocValuesType.BINARY+ && t != DocValuesType.SORTED_NUMERIC) {continue;}FieldInfo baseFi = baseFieldInfos.fieldInfo(fi.name);@@ -4609,7 +4650,7 @@ public class IndexWriterp =u.type == DocValuesType.BINARY? new BinaryDocValuesFieldUpdates(u.delGen, u.field, mergedMaxDoc)- : new NumericDocValuesFieldUpdates(u.delGen, u.field, mergedMaxDoc);+ : new NumericDocValuesFieldUpdates(u.delGen, u.field, u.type, mergedMaxDoc);byGen.put(u.delGen, p);}if (it.hasValue()) {@@ -4639,7 +4680,8 @@ public class IndexWriterp =fi.getDocValuesType() == DocValuesType.BINARY? new BinaryDocValuesFieldUpdates(delGen, fi.name, mergedMaxDoc)- : new NumericDocValuesFieldUpdates(delGen, fi.name, mergedMaxDoc);+ : new NumericDocValuesFieldUpdates(+ delGen, fi.name, fi.getDocValuesType(), mergedMaxDoc);byGen.put(delGen, p);}return p;@@ -4659,51 +4701,130 @@ public class IndexWriterint mergedMaxDoc,LongObjectHashMap<DocValuesFieldUpdates> byGen)throws IOException {+ switch (fi.getDocValuesType()) {+ case NUMERIC ->+ addNumericDiskDiffToPacket(+ fi, baseline, current, segDocMap, diskDelGen, mergedMaxDoc, byGen);+ case BINARY ->+ addBinaryDiskDiffToPacket(+ fi, baseline, current, segDocMap, diskDelGen, mergedMaxDoc, byGen);+ case SORTED_NUMERIC ->+ addSortedNumericDiskDiffToPacket(+ fi, baseline, current, segDocMap, diskDelGen, mergedMaxDoc, byGen);+ // $CASES-OMITTED$+ default ->+ throw new AssertionError(+ "unexpected doc-values type for update carry-over: " + fi.getDocValuesType());+ }+ }++ private static void addNumericDiskDiffToPacket(+ FieldInfo fi,+ CodecReader baseline,+ CodecReader current,+ MergeState.DocMap segDocMap,+ long diskDelGen,+ int mergedMaxDoc,+ LongObjectHashMap<DocValuesFieldUpdates> byGen)+ throws IOException {final int maxDoc = current.maxDoc();- final boolean binary = fi.getDocValuesType() == DocValuesType.BINARY;- BinaryDocValues curB = binary ? current.getBinaryDocValues(fi.name) : null;- BinaryDocValues baseB = binary ? baseline.getBinaryDocValues(fi.name) : null;- NumericDocValues curN = binary ? null : current.getNumericDocValues(fi.name);- NumericDocValues baseN = binary ? null : baseline.getNumericDocValues(fi.name);+ NumericDocValues cur = current.getNumericDocValues(fi.name);+ NumericDocValues base = baseline.getNumericDocValues(fi.name);for (int doc = 0; doc < maxDoc; doc++) {int md = segDocMap.get(doc);if (md == -1) {continue;}- Object curVal;- Object baseVal;- if (binary) {- // Live BytesRefs suffice to compare; only a changed value is copied, below, when it is- // stored.- curVal = (curB != null && curB.advanceExact(doc)) ? curB.binaryValue() : DV_NO_VALUE;- baseVal = (baseB != null && baseB.advanceExact(doc)) ? baseB.binaryValue() : DV_NO_VALUE;+ boolean hasCur = cur != null && cur.advanceExact(doc);+ boolean hasBase = base != null && base.advanceExact(doc);+ long curVal = hasCur ? cur.longValue() : 0;+ if (hasCur == hasBase && (hasCur == false || curVal == base.longValue())) {+ continue; // unchanged: both absent, or both present with an equal value+ }+ DocValuesFieldUpdates p = getOrCreatePacket(byGen, diskDelGen, fi, mergedMaxDoc);+ if (hasCur == false) {+ p.reset(md);} else {- curVal =- (curN != null && curN.advanceExact(doc)) ? Long.valueOf(curN.longValue()) : DV_NO_VALUE;- baseVal =- (baseN != null && baseN.advanceExact(doc))- ? Long.valueOf(baseN.longValue())- : DV_NO_VALUE;- }- if (dvEquals(curVal, baseVal)) {+ p.add(md, curVal);+ }+ }+ }++ private static void addBinaryDiskDiffToPacket(+ FieldInfo fi,+ CodecReader baseline,+ CodecReader current,+ MergeState.DocMap segDocMap,+ long diskDelGen,+ int mergedMaxDoc,+ LongObjectHashMap<DocValuesFieldUpdates> byGen)+ throws IOException {+ final int maxDoc = current.maxDoc();+ BinaryDocValues cur = current.getBinaryDocValues(fi.name);+ BinaryDocValues base = baseline.getBinaryDocValues(fi.name);+ for (int doc = 0; doc < maxDoc; doc++) {+ int md = segDocMap.get(doc);+ if (md == -1) {continue;}+ // cur and base are independent readers, so their live BytesRefs can be compared directly; a+ // changed value is deep-copied only when it is stored below.+ boolean hasCur = cur != null && cur.advanceExact(doc);+ BytesRef curVal = hasCur ? cur.binaryValue() : null;+ boolean hasBase = base != null && base.advanceExact(doc);+ if (hasCur == hasBase && (hasCur == false || curVal.bytesEquals(base.binaryValue()))) {+ continue; // unchanged: both absent, or both present with an equal value+ }DocValuesFieldUpdates p = getOrCreatePacket(byGen, diskDelGen, fi, mergedMaxDoc);- if (curVal == DV_NO_VALUE) {+ if (hasCur == false) {p.reset(md);- } else if (binary) {- ((BinaryDocValuesFieldUpdates) p).add(md, BytesRef.deepCopyOf((BytesRef) curVal));} else {- ((NumericDocValuesFieldUpdates) p).add(md, (Long) curVal);+ p.add(md, BytesRef.deepCopyOf(curVal));}}}- private static boolean dvEquals(Object a, Object b) {- if (a == DV_NO_VALUE || b == DV_NO_VALUE) {- return a == b;+ /**+ * The sorted-numeric analog: a doc whose whole value set changed during the merge was updated,+ * and an update always sets a single value (an unchanged multi-valued doc compares equal and is+ * skipped), so a changed doc is emitted as either a single value or a reset.+ */+ private static void addSortedNumericDiskDiffToPacket(+ FieldInfo fi,+ CodecReader baseline,+ CodecReader current,+ MergeState.DocMap segDocMap,+ long diskDelGen,+ int mergedMaxDoc,+ LongObjectHashMap<DocValuesFieldUpdates> byGen)+ throws IOException {+ final int maxDoc = current.maxDoc();+ SortedNumericDocValues cur = current.getSortedNumericDocValues(fi.name);+ SortedNumericDocValues base = baseline.getSortedNumericDocValues(fi.name);+ for (int doc = 0; doc < maxDoc; doc++) {+ int md = segDocMap.get(doc);+ if (md == -1) {+ continue;+ }+ boolean hasCur = cur != null && cur.advanceExact(doc);+ if (hasCur && cur.docValueCount() > 1) {+ // Updates are always single-valued, so a still-multi-valued doc cannot have been updated.+ continue;+ }+ long curVal = hasCur ? cur.nextValue() : 0;+ boolean hasBase = base != null && base.advanceExact(doc);+ if (hasCur == false && hasBase == false) {+ continue;+ } else if (hasCur && hasBase && base.docValueCount() == 1 && base.nextValue() == curVal) {+ continue;+ }+ DocValuesFieldUpdates p = getOrCreatePacket(byGen, diskDelGen, fi, mergedMaxDoc);+ if (hasCur == false) {+ p.reset(md);+ } else {+ p.add(md, curVal);+ }}- return a.equals(b);}/**diff --git a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.javaindex 41b642ddf59..edeec613ff0 100644--- a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java+++ b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java@@ -67,7 +67,11 @@ final class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {private final long minValue;NumericDocValuesFieldUpdates(long delGen, String field, int maxDoc) {- super(maxDoc, delGen, field, DocValuesType.NUMERIC);+ this(delGen, field, DocValuesType.NUMERIC, maxDoc);+ }++ NumericDocValuesFieldUpdates(long delGen, String field, DocValuesType type, int maxDoc) {+ super(maxDoc, delGen, field, type);// we don't know the min/max range so we use the growable writer here to adjust as we go.values = new PagedGrowableWriter(1, PAGE_SIZE, 1, PackedInts.DEFAULT);minValue = 0;@@ -75,7 +79,12 @@ final class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {NumericDocValuesFieldUpdates(long delGen, String field, long minValue, long maxValue, int maxDoc) {- super(maxDoc, delGen, field, DocValuesType.NUMERIC);+ this(delGen, field, DocValuesType.NUMERIC, minValue, maxValue, maxDoc);+ }++ NumericDocValuesFieldUpdates(+ long delGen, String field, DocValuesType type, long minValue, long maxValue, int maxDoc) {+ super(maxDoc, delGen, field, type);assert minValue <= maxValue: "minValue must be <= maxValue [" + minValue + " > " + maxValue + "]";int bitsPerValue = PackedInts.unsignedBitsRequired(maxValue - minValue);@@ -141,7 +150,12 @@ final class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {private boolean hasAtLeastOneValue;SingleValueNumericDocValuesFieldUpdates(long delGen, String field, int maxDoc, long value) {- super(maxDoc, delGen, field, DocValuesType.NUMERIC);+ this(delGen, field, DocValuesType.NUMERIC, maxDoc, value);+ }++ SingleValueNumericDocValuesFieldUpdates(+ long delGen, String field, DocValuesType type, int maxDoc, long value) {+ super(maxDoc, delGen, field, type);this.bitSet = new SparseFixedBitSet(maxDoc);this.value = value;}diff --git a/lucene/core/src/java/org/apache/lucene/index/OverlaySortedNumericDocValues.java b/lucene/core/src/java/org/apache/lucene/index/OverlaySortedNumericDocValues.javanew file mode 100644index 00000000000..d7d8a9bebb4--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/index/OverlaySortedNumericDocValues.java@@ -0,0 +1,94 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.index;++import java.io.IOException;+import java.util.List;+import org.apache.lucene.codecs.DocValuesProducer;++/**+ * Merges several {@link SortedNumericDocValues} layers (newest generation first, base last) into+ * one view: for a given document the whole value set comes from the newest layer that has it, else+ * the base. Unlike {@link OverlayNumericDocValues} this preserves multi-valued documents, so it+ * works whether the base column is single- or multi-valued (a set-only sorted-numeric update writes+ * a single-valued delta on top of a possibly multi-valued base).+ */+final class OverlaySortedNumericDocValues extends SortedNumericDocValues {++ private final SortedNumericDocValues[] layers; // newest first, base last+ private final DocValuesOverlayMerger merger;++ OverlaySortedNumericDocValues(SortedNumericDocValues[] layers) {+ assert layers != null && layers.length > 0;+ this.layers = layers;+ this.merger = new DocValuesOverlayMerger(layers);+ }++ @Override+ public long nextValue() throws IOException {+ return layers[merger.valueLayer()].nextValue();+ }++ @Override+ public int docValueCount() {+ return layers[merger.valueLayer()].docValueCount();+ }++ @Override+ public boolean advanceExact(int target) throws IOException {+ return merger.advanceExact(target);+ }++ @Override+ public int docID() {+ return merger.docID();+ }++ @Override+ public int nextDoc() throws IOException {+ return merger.nextDoc();+ }++ @Override+ public int advance(int target) throws IOException {+ return merger.advance(target);+ }++ @Override+ public long cost() {+ return merger.cost();+ }++ static SortedNumericDocValues from(FieldInfo fieldInfo, List<DocValuesProducer> producers)+ throws IOException {+ SortedNumericDocValues[] layers = new SortedNumericDocValues[producers.size()];+ boolean allSingleValued = true;+ for (int i = 0; i < layers.length; i++) {+ layers[i] = producers.get(i).getSortedNumeric(fieldInfo);+ allSingleValued &= DocValues.isSingleton(layers[i]);+ }+ if (allSingleValued) {+ NumericDocValues[] singletonLayers = new NumericDocValues[producers.size()];+ for (int i = 0; i < layers.length; i++) {+ singletonLayers[i] = DocValues.unwrapSingleton(layers[i]);+ }+ return DocValues.singleton(new OverlayNumericDocValues(singletonLayers));+ } else {+ return new OverlaySortedNumericDocValues(layers);+ }+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.javaindex 80c062bbd42..4117d3bbec8 100644--- a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java+++ b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java@@ -305,7 +305,9 @@ final class ReadersAndUpdates {final String field = ent.getKey();final List<DocValuesFieldUpdates> updates = ent.getValue();DocValuesType type = updates.get(0).type;- assert type == DocValuesType.NUMERIC || type == DocValuesType.BINARY+ assert type == DocValuesType.NUMERIC+ || type == DocValuesType.BINARY+ || type == DocValuesType.SORTED_NUMERIC: "unsupported type: " + type;final List<DocValuesFieldUpdates> updatesToApply = new ArrayList<>();long bytes = 0;@@ -354,9 +356,10 @@ final class ReadersAndUpdates {}}// Set-only updates become a sparse delta generation overlaid at read time; a removal falls- // back to the dense- // rewrite. (Skip-indexed fields can't reach here: IndexWriter rejects doc-values updates on- // them.)+ // back to the dense rewrite. (Skip-indexed fields can't reach here: IndexWriter rejects+ // doc-values updates on them.) A single-valued sorted-numeric update writes a single-valued+ // delta; the base column may be single- or multi-valued and is preserved per-doc by the+ // sorted-numeric merge/overlay.final boolean sparseDelta = config.getMaxDocValuesOverlays() > 0 && anyRemoval == false;// Past maxDocValuesOverlays, fold the prior deltas into this write as one sparse generation// (base untouched) instead of appending another.@@ -384,9 +387,12 @@ final class ReadersAndUpdates {// maxDoc for sparse columns) and deltas are small by construction.deltaCoverage +=docsWithValueCount(- type == DocValuesType.BINARY- ? p.getBinary(fieldInfo)- : p.getNumeric(fieldInfo));+ switch (type) {+ case BINARY -> p.getBinary(fieldInfo);+ case SORTED_NUMERIC -> p.getSortedNumeric(fieldInfo);+ // $CASES-OMITTED$+ default -> p.getNumeric(fieldInfo);+ });}} catch (Throwable t) {// The try-with-resources below owns the normal close; close here if opening a delta fails@@ -483,7 +489,7 @@ final class ReadersAndUpdates {};}});- } else {+ } else if (type == DocValuesType.NUMERIC) {// write the numeric updates to a new gen'd docvalues filefieldsConsumer.addNumericField(fieldInfo,@@ -501,49 +507,87 @@ final class ReadersAndUpdates {compact && foldToDense == false? overlayNumeric(deltaProducers, fieldInfo): reader.getNumericDocValues(field);- final MergedDocValues<NumericDocValues> mergedDocValues =- new MergedDocValues<>(- onDisk,- DocValuesFieldUpdates.Iterator.asNumericDocValues(iterator),- iterator);- // Merge sort of the original doc values with updated doc values:- return new NumericDocValues() {- @Override- public long longValue() throws IOException {- return mergedDocValues.currentValuesSupplier.longValue();- }-- @Override- public boolean advanceExact(int target) {- return mergedDocValues.advanceExact(target);- }-- @Override- public int docID() {- return mergedDocValues.docID();- }-- @Override- public int nextDoc() throws IOException {- return mergedDocValues.nextDoc();- }-- @Override- public int advance(int target) {- return mergedDocValues.advance(target);- }-- @Override- public void intoBitSet(int upTo, FixedBitSet bitSet, int offset)- throws IOException {- mergedDocValues.intoBitSet(upTo, bitSet, offset);- }-- @Override- public long cost() {- return mergedDocValues.cost();- }- };+ return handleMergedNumericDocValues(onDisk, iterator);+ }+ });+ } else {+ assert type == DocValuesType.SORTED_NUMERIC : "unsupported type: " + type;+ // The update is single-valued but the base column may be multi-valued, so merge/overlay+ // natively as sorted-numeric (the update wins per doc, other docs keep their whole set).+ fieldsConsumer.addSortedNumericField(+ fieldInfo,+ new EmptyDocValuesProducer() {+ @Override+ public SortedNumericDocValues getSortedNumeric(FieldInfo fieldInfoIn)+ throws IOException {+ DocValuesFieldUpdates.Iterator iterator = updateSupplier.apply(fieldInfo);+ if (sparseDelta && compact == false) {+ // write only the updated docs (single-valued) as a singleton column; unchanged+ // docs come from the base at read time+ return DocValues.singleton(+ DocValuesFieldUpdates.Iterator.asNumericDocValues(iterator));+ }+ // sparse fold merges the prior deltas (stays sparse); a dense rewrite (removal or+ // fold-to-dense) merges the base column+ final SortedNumericDocValues onDisk =+ compact && foldToDense == false+ ? overlaySortedNumeric(deltaProducers, fieldInfo)+ : reader.getSortedNumericDocValues(field);+ final NumericDocValues singletonOnDisk = DocValues.unwrapSingleton(onDisk);+ if (singletonOnDisk != null) {+ return DocValues.singleton(+ handleMergedNumericDocValues(singletonOnDisk, iterator));+ } else {+ final MergedDocValues<SortedNumericDocValues> mergedDocValues =+ new MergedDocValues<>(+ onDisk,+ DocValues.singleton(+ DocValuesFieldUpdates.Iterator.asNumericDocValues(iterator)),+ iterator);+ // Merge sort of the original doc values with updated doc values:+ return new SortedNumericDocValues() {+ @Override+ public long nextValue() throws IOException {+ return mergedDocValues.currentValuesSupplier.nextValue();+ }++ @Override+ public int docValueCount() {+ return mergedDocValues.currentValuesSupplier.docValueCount();+ }++ @Override+ public boolean advanceExact(int target) {+ return mergedDocValues.advanceExact(target);+ }++ @Override+ public int docID() {+ return mergedDocValues.docID();+ }++ @Override+ public int nextDoc() throws IOException {+ return mergedDocValues.nextDoc();+ }++ @Override+ public int advance(int target) {+ return mergedDocValues.advance(target);+ }++ @Override+ public void intoBitSet(int upTo, FixedBitSet bitSet, int offset)+ throws IOException {+ mergedDocValues.intoBitSet(upTo, bitSet, offset);+ }++ @Override+ public long cost() {+ return mergedDocValues.cost();+ }+ };+ }}});}@@ -605,6 +649,15 @@ final class ReadersAndUpdates {return new OverlayBinaryDocValues(layers);}+ /**+ * Fresh overlay over the delta generations of a sorted-numeric field, newest first, used to fold+ * deltas during compaction.+ */+ private static SortedNumericDocValues overlaySortedNumeric(+ List<DocValuesProducer> producers, FieldInfo fieldInfo) throws IOException {+ return OverlaySortedNumericDocValues.from(fieldInfo, producers);+ }+/*** This class merges the current on-disk DV with an incoming update DV instance and merges the two* instances giving the incoming update precedence in terms of values, in other words the values@@ -741,6 +794,52 @@ final class ReadersAndUpdates {}}+ private NumericDocValues handleMergedNumericDocValues(+ NumericDocValues onDisk, DocValuesFieldUpdates.Iterator updateIterator) {+ final MergedDocValues<NumericDocValues> mergedDocValues =+ new MergedDocValues<>(+ onDisk,+ DocValuesFieldUpdates.Iterator.asNumericDocValues(updateIterator),+ updateIterator);+ // Merge sort of the original doc values with updated doc values:+ return new NumericDocValues() {+ @Override+ public long longValue() throws IOException {+ return mergedDocValues.currentValuesSupplier.longValue();+ }++ @Override+ public boolean advanceExact(int target) {+ return mergedDocValues.advanceExact(target);+ }++ @Override+ public int docID() {+ return mergedDocValues.docID();+ }++ @Override+ public int nextDoc() throws IOException {+ return mergedDocValues.nextDoc();+ }++ @Override+ public int advance(int target) {+ return mergedDocValues.advance(target);+ }++ @Override+ public void intoBitSet(int upTo, FixedBitSet bitSet, int offset) throws IOException {+ mergedDocValues.intoBitSet(upTo, bitSet, offset);+ }++ @Override+ public long cost() {+ return mergedDocValues.cost();+ }+ };+ }+private synchronized Set<String> writeFieldInfosGen(FieldInfos fieldInfos, Directory dir, FieldInfosFormat infosFormat) throws IOException {final long nextFieldInfosGen = info.getNextFieldInfosGen();diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.javaindex b3979856745..c20171c17e7 100644--- a/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java@@ -17,6 +17,7 @@package org.apache.lucene.index;import java.io.IOException;+import java.util.Arrays;import java.util.Collections;import java.util.HashMap;import java.util.IdentityHashMap;@@ -199,7 +200,7 @@ class SegmentDocValuesProducer extends DocValuesProducer {return new OverlayBinaryDocValues(layers);}- // sorted/sorted-set/sorted-numeric and skippers are never overlaid: only numeric/binary values+ // sorted/sorted-set and skippers are never overlaid: only numeric/binary/sorted-numeric values// can be updated in place, so these always have a single producer.private DocValuesProducer single(FieldInfo field) {DocValuesProducer[] producers = dvProducersByField.get(field.number);@@ -215,7 +216,12 @@ class SegmentDocValuesProducer extends DocValuesProducer {@Overridepublic SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {- return single(field).getSortedNumeric(field);+ DocValuesProducer[] producers = dvProducersByField.get(field.number);+ assert producers != null;+ if (producers.length == 1) {+ return producers[0].getSortedNumeric(field);+ }+ return OverlaySortedNumericDocValues.from(field, Arrays.asList(producers));}@Overridediff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.javaindex a1a1396a3dd..4c41d4e62b1 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java@@ -2017,6 +2017,13 @@ public class TestIndexSorting extends LuceneTestCase {assertEquals(exc.getMessage(),"cannot update docvalues field involved in the index sort, field=foo, sort=<long: \"foo\">");+ exc =+ expectThrows(+ IllegalArgumentException.class,+ () -> w.updateSortedNumericDocValue(new Term("id", "0"), "foo", -1));+ assertEquals(+ exc.getMessage(),+ "cannot update docvalues field involved in the index sort, field=foo, sort=<long: \"foo\">");w.close();dir.close();}diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMergeCarryOverFromDisk.java b/lucene/core/src/test/org/apache/lucene/index/TestMergeCarryOverFromDisk.javaindex 0cf456aa53e..46993a71891 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestMergeCarryOverFromDisk.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestMergeCarryOverFromDisk.java@@ -17,12 +17,19 @@package org.apache.lucene.index;import java.io.IOException;+import java.util.Arrays;+import java.util.HashMap;+import java.util.HashSet;+import java.util.Map;+import java.util.Set;import java.util.concurrent.CountDownLatch;import java.util.concurrent.atomic.AtomicReference;import org.apache.lucene.document.BinaryDocValuesField;import org.apache.lucene.document.Document;import org.apache.lucene.document.NumericDocValuesField;+import org.apache.lucene.document.SortedNumericDocValuesField;import org.apache.lucene.document.StringField;+import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.store.Directory;import org.apache.lucene.store.FilterDirectory;import org.apache.lucene.store.IOContext;@@ -67,10 +74,19 @@ public class TestMergeCarryOverFromDisk extends LuceneTestCase {}private static Document doc(int id, long ndv, String bdv) {+ return doc(id, ndv, bdv, ndv);+ }++ /** As {@link #doc(int, long, String)} but with an explicit, possibly multi-valued, snv set. */+ private static Document doc(int id, long ndv, String bdv, long... snv) {Document d = new Document();d.add(new StringField("id", Integer.toString(id), StringField.Store.NO));d.add(new NumericDocValuesField("ndv", ndv));d.add(new BinaryDocValuesField("bdv", new BytesRef(bdv)));+ d.add(new SortedNumericDocValuesField("svsnv", ndv));+ for (long v : snv) {+ d.add(new SortedNumericDocValuesField("snv", v));+ }return d;}@@ -81,13 +97,17 @@ public class TestMergeCarryOverFromDisk extends LuceneTestCase {.setMergeScheduler(new ConcurrentMergeScheduler());IndexWriter writer = new IndexWriter(dir, conf);- // Two segments; the default TieredMergePolicy won't auto-merge two small segments.+ // Two segments; the default TieredMergePolicy won't auto-merge two small segments. docs 0-3 are+ // single-valued and should create a NumericDocValues on disk; docs 4-7 are genuinely+ // multi-valued ({i, i+1000}) and should create a SortedNumericDocValues on disk. A doc never+ // updated for snv must keep its whole value set through the merge, while a doc updated for snv+ // (3 and 7 below) collapses to the single update value.for (int i = 0; i < 4; i++) {- writer.addDocument(doc(i, i, "v" + i));+ writer.addDocument(doc(i, i, "v" + i, i));}writer.commit();for (int i = 4; i < 8; i++) {- writer.addDocument(doc(i, i, "v" + i));+ writer.addDocument(doc(i, i, "v" + i, i, i + 1000));}writer.commit();@@ -109,12 +129,16 @@ public class TestMergeCarryOverFromDisk extends LuceneTestCase {// output.dir.mergeStarted.await();- // Update numeric + binary values on docs in the now-merging segments, then resolve them to the- // segments (and disk) by reopening a reader from the writer.+ // Update numeric, binary, and sorted-numeric values on docs in the now-merging segments, then+ // resolve them to the segments (and disk) by reopening a reader from the writer.writer.updateNumericDocValue(new Term("id", "1"), "ndv", 101L);writer.updateNumericDocValue(new Term("id", "5"), "ndv", 105L);writer.updateBinaryDocValue(new Term("id", "2"), "bdv", new BytesRef("updated-2"));writer.updateBinaryDocValue(new Term("id", "6"), "bdv", new BytesRef("updated-6"));+ writer.updateSortedNumericDocValue(new Term("id", "3"), "snv", 103L);+ writer.updateSortedNumericDocValue(new Term("id", "7"), "snv", 107L);+ writer.updateSortedNumericDocValue(new Term("id", "3"), "svsnv", 103L);+ writer.updateSortedNumericDocValue(new Term("id", "7"), "svsnv", 107L);// Force resolution onto the merging segments (writes DV gens to disk).try (DirectoryReader r = DirectoryReader.open(writer)) {assertNotNull(r);@@ -125,25 +149,76 @@ public class TestMergeCarryOverFromDisk extends LuceneTestCase {merger.join();assertNull("forceMerge failed: " + dir.failure.get(), dir.failure.get());- // Verify the merged segment reflects the numeric + binary updates that resolved during the- // merge.+ // Verify every doc in the merged segment: the numeric, binary, and sorted-numeric updates that+ // resolved during the merge landed on exactly their docs, and every untouched doc (including+ // the+ // multi-valued one) kept the value it had before the merge.try (DirectoryReader reader = DirectoryReader.open(writer)) {assertEquals(1, reader.leaves().size()); // single merged segmentLeafReader leaf = reader.leaves().get(0).reader();NumericDocValues ndv = leaf.getNumericDocValues("ndv");BinaryDocValues bdv = leaf.getBinaryDocValues("bdv");- java.util.Set<Long> ndvValues = new java.util.HashSet<>();- java.util.Set<String> bdvValues = new java.util.HashSet<>();+ SortedNumericDocValues svsnv = leaf.getSortedNumericDocValues("svsnv");+ assertTrue(+ "single-valued SortedNumericDocValues should be a singleton",+ DocValues.isSingleton(svsnv));+ SortedNumericDocValues snv = leaf.getSortedNumericDocValues("snv");+ assertFalse(+ "multi-valued SortedNumericDocValues should not be a singleton",+ DocValues.isSingleton(snv));+ // docId -> id, so per-doc sorted-numeric value sets can be checked by id (postings iterate+ // per term, not in doc order).+ Map<Integer, String> idByDoc = new HashMap<>();+ TermsEnum te = leaf.terms("id").iterator();+ BytesRef t;+ while ((t = te.next()) != null) {+ PostingsEnum pe = te.postings(null, PostingsEnum.NONE);+ for (int d = pe.nextDoc(); d != DocIdSetIterator.NO_MORE_DOCS; d = pe.nextDoc()) {+ idByDoc.put(d, t.utf8ToString());+ }+ }+ Map<String, Long> ndvById = new HashMap<>();+ Map<String, String> bdvById = new HashMap<>();+ Map<String, long[]> snvById = new HashMap<>();for (int d = 0; d < leaf.maxDoc(); d++) {+ String id = idByDoc.get(d);assertTrue(ndv.advanceExact(d));- ndvValues.add(ndv.longValue());+ ndvById.put(id, ndv.longValue());assertTrue(bdv.advanceExact(d));- bdvValues.add(bdv.binaryValue().utf8ToString());+ bdvById.put(id, bdv.binaryValue().utf8ToString());+ assertTrue(snv.advanceExact(d)); // doc order, so advanceExact is monotonic+ long[] vals = new long[snv.docValueCount()];+ for (int i = 0; i < vals.length; i++) {+ vals[i] = snv.nextValue();+ }+ snvById.put(id, vals);+ }+ // Every doc must hold exactly its expected value after the merge: the four updated docs carry+ // the new value, and every untouched doc keeps what it had before the merge.+ for (int id = 0; id < 8; id++) {+ String key = Integer.toString(id);+ long expectedNdv =+ switch (id) {+ case 1 -> 101L;+ case 5 -> 105L;+ default -> id;+ };+ assertEquals("ndv for doc " + id, expectedNdv, (long) ndvById.get(key));+ String expectedBdv =+ switch (id) {+ case 2 -> "updated-2";+ case 6 -> "updated-6";+ default -> "v" + id;+ };+ assertEquals("bdv for doc " + id, expectedBdv, bdvById.get(key));+ long[] expectedSnv =+ switch (id) {+ case 3 -> new long[] {103};+ case 7 -> new long[] {107};+ default -> id < 4 ? new long[] {id} : new long[] {id, id + 1000};+ };+ assertArrayEquals("snv for doc " + id, expectedSnv, snvById.get(key));}- assertTrue("carried numeric update 101 missing", ndvValues.contains(101L));- assertTrue("carried numeric update 105 missing", ndvValues.contains(105L));- assertTrue("carried binary update updated-2 missing", bdvValues.contains("updated-2"));- assertTrue("carried binary update updated-6 missing", bdvValues.contains("updated-6"));}writer.close();@@ -163,12 +238,14 @@ public class TestMergeCarryOverFromDisk extends LuceneTestCase {// small overlay budget so a handful of updates already triggers a fold.setMaxDocValuesOverlays(4);IndexWriter writer = new IndexWriter(dir, conf);+ // Segment 1 (docs 0-3): single-valued snv, stored as a numeric column. Segment 2 (docs 4-7):+ // multi-valued snv ({i, i+1000}), stored as a sorted-numeric column.for (int i = 0; i < 4; i++) {- writer.addDocument(doc(i, i, "v" + i));+ writer.addDocument(doc(i, i, "v" + i, i));}writer.commit();for (int i = 4; i < 8; i++) {- writer.addDocument(doc(i, i, "v" + i));+ writer.addDocument(doc(i, i, "v" + i, i, i + 1000));}writer.commit();@@ -186,12 +263,16 @@ public class TestMergeCarryOverFromDisk extends LuceneTestCase {dir.mergeStarted.await();// Many updates to doc 3 (in the first, merging segment), each resolved separately so it becomes- // a- // new overlay generation; well past maxDocValuesOverlays, so folds happen mid-merge.+ // a new overlay generation; well past maxDocValuesOverlays, so folds happen mid-merge. Both a+ // numeric and a sorted-numeric field are folded so the carry-over reads back a folded column of+ // each kind.long finalValue = -1;+ long finalSnv = -1;for (int v = 0; v < 20; v++) {finalValue = 1000L + v;+ finalSnv = 2000L + v;writer.updateNumericDocValue(new Term("id", "3"), "ndv", finalValue);+ writer.updateSortedNumericDocValue(new Term("id", "3"), "snv", finalSnv);try (DirectoryReader r = DirectoryReader.open(writer)) {assertNotNull(r);}@@ -203,13 +284,21 @@ public class TestMergeCarryOverFromDisk extends LuceneTestCase {try (DirectoryReader reader = DirectoryReader.open(writer)) {assertEquals(1, reader.leaves().size());- NumericDocValues ndv = reader.leaves().get(0).reader().getNumericDocValues("ndv");- java.util.Set<Long> values = new java.util.HashSet<>();- for (int d = 0; d < reader.leaves().get(0).reader().maxDoc(); d++) {+ LeafReader leaf = reader.leaves().get(0).reader();+ NumericDocValues ndv = leaf.getNumericDocValues("ndv");+ SortedNumericDocValues snv = leaf.getSortedNumericDocValues("snv");+ Set<Long> values = new HashSet<>();+ Set<Long> snvValues = new HashSet<>();+ for (int d = 0; d < leaf.maxDoc(); d++) {assertTrue(ndv.advanceExact(d));values.add(ndv.longValue());+ assertTrue(snv.advanceExact(d));+ for (int i = 0, count = snv.docValueCount(); i < count; i++) {+ snvValues.add(snv.nextValue());+ }}assertTrue("folded final value " + finalValue + " missing", values.contains(finalValue));+ assertTrue("folded final snv " + finalSnv + " missing", snvValues.contains(finalSnv));}writer.close();@@ -229,12 +318,14 @@ public class TestMergeCarryOverFromDisk extends LuceneTestCase {.setMergeScheduler(new ConcurrentMergeScheduler()).setSoftDeletesField("__soft");IndexWriter writer = new IndexWriter(dir, conf);+ // Segment 1 (docs 0-3): single-valued snv, stored as a numeric column. Segment 2 (docs 4-7):+ // multi-valued snv ({i, i+1000}), stored as a sorted-numeric column.for (int i = 0; i < 4; i++) {- writer.addDocument(doc(i, i, "v" + i));+ writer.addDocument(doc(i, i, "v" + i, i));}writer.commit();for (int i = 4; i < 8; i++) {- writer.addDocument(doc(i, i, "v" + i));+ writer.addDocument(doc(i, i, "v" + i, i, i + 1000));}writer.commit();@@ -256,6 +347,10 @@ public class TestMergeCarryOverFromDisk extends LuceneTestCase {Document replacement = doc(1, 101, "updated-1");writer.softUpdateDocument(new Term("id", "1"), replacement, new NumericDocValuesField("__soft", 1));+ // Also update a sorted-numeric value on a surviving multi-valued doc (id 5, in segment 2), to+ // confirm the update collapses it to a single value and is carried over alongside the+ // soft-delete.+ writer.updateSortedNumericDocValue(new Term("id", "5"), "snv", 205L);try (DirectoryReader r = DirectoryReader.open(writer)) {assertNotNull(r);}@@ -265,23 +360,39 @@ public class TestMergeCarryOverFromDisk extends LuceneTestCase {assertNull("forceMerge failed: " + dir.failure.get(), dir.failure.get());// Exactly one doc with id=1 must be live (the replacement); the old one must be soft-deleted.+ // The replacement doc carries its own snv, and the sorted-numeric update on doc 5 is carried+ // over too.try (DirectoryReader reader = DirectoryReader.open(writer)) {int live = 0;+ boolean sawCarriedSnv = false;for (LeafReaderContext ctx : reader.leaves()) {LeafReader leaf = ctx.reader();NumericDocValues ndv = leaf.getNumericDocValues("ndv");+ SortedNumericDocValues snv = leaf.getSortedNumericDocValues("snv");Bits liveDocs = leaf.getLiveDocs();for (int d = 0; d < leaf.maxDoc(); d++) {if (liveDocs != null && liveDocs.get(d) == false) {continue;}assertTrue(ndv.advanceExact(d));+ assertTrue(snv.advanceExact(d)); // doc order, so advanceExact is monotonic+ long[] snvVals = new long[snv.docValueCount()];+ for (int i = 0; i < snvVals.length; i++) {+ snvVals[i] = snv.nextValue();+ }if (ndv.longValue() == 101) {live++;+ // the replacement doc is in the single-valued segment+ assertArrayEquals("replacement doc snv", new long[] {101}, snvVals);+ }+ // doc 5 (multi-valued base) collapsed to a single value by the carried update+ if (Arrays.equals(snvVals, new long[] {205})) {+ sawCarriedSnv = true;}}}assertEquals("exactly one live replacement doc expected", 1, live);+ assertTrue("carried sorted-numeric update 205 missing", sawCarriedSnv);}writer.close();diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSortedNumericDocValuesUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestSortedNumericDocValuesUpdates.javanew file mode 100644index 00000000000..b98e1eb1d1a--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/index/TestSortedNumericDocValuesUpdates.java@@ -0,0 +1,669 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.index;++import java.util.Arrays;+import java.util.HashMap;+import java.util.Map;+import java.util.TreeMap;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.Field.Store;+import org.apache.lucene.document.SortedNumericDocValuesField;+import org.apache.lucene.document.StringField;+import org.apache.lucene.search.DocIdSetIterator;+import org.apache.lucene.search.Sort;+import org.apache.lucene.search.SortField;+import org.apache.lucene.search.SortedNumericSortField;+import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.analysis.MockAnalyzer;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.Bits;+import org.apache.lucene.util.IOUtils;++/**+ * Tests in-place updates of SORTED_NUMERIC doc-values. An update sets a single value per matched+ * document, replacing whatever value(s) that document had; the field itself may be single- or+ * multi-valued, and documents not matched by the update keep their existing values. A single-valued+ * column is stored as a numeric column and reuses the numeric update machinery; a multi-valued+ * column is preserved and merged per-doc.+ */+public class TestSortedNumericDocValuesUpdates extends LuceneTestCase {++ private Document doc(int id, long... values) {+ Document doc = new Document();+ doc.add(new StringField("id", "doc-" + id, Store.NO));+ for (long v : values) {+ doc.add(new SortedNumericDocValuesField("val", v));+ }+ return doc;+ }++ private IndexWriterConfig conf() {+ IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));+ // exercise both the classic dense rewrite and the sparse-delta overlay+ conf.setMaxDocValuesOverlays(random().nextBoolean() ? 0 : 1 + random().nextInt(4));+ return conf;+ }++ public void testSimpleReplace() throws Exception {+ Directory dir = newDirectory();+ IndexWriter writer = new IndexWriter(dir, conf());+ writer.addDocument(doc(0, 3));+ writer.addDocument(doc(1, 5));+ writer.commit();++ writer.updateSortedNumericDocValue(new Term("id", "doc-0"), "val", 9);+ try (DirectoryReader reader = DirectoryReader.open(writer)) {+ Map<String, long[]> values = collect(reader);+ assertArrayEquals(new long[] {9}, values.get("doc-0"));+ assertArrayEquals(new long[] {5}, values.get("doc-1"));+ }+ IOUtils.close(writer, dir);+ }++ public void testUpdateViaFieldsApi() throws Exception {+ Directory dir = newDirectory();+ IndexWriter writer = new IndexWriter(dir, conf());+ writer.addDocument(doc(0, 1));+ writer.commit();++ writer.updateDocValues(new Term("id", "doc-0"), new SortedNumericDocValuesField("val", 42));+ try (DirectoryReader reader = DirectoryReader.open(writer)) {+ assertArrayEquals(new long[] {42}, collect(reader).get("doc-0"));+ }+ IOUtils.close(writer, dir);+ }++ public void testMultipleFieldsSameNameLastWins() throws Exception {+ Directory dir = newDirectory();+ IndexWriter writer = new IndexWriter(dir, conf());+ writer.addDocument(doc(0, 1));+ writer.commit();+ // two same-named single-valued sorted-numeric fields in one update behave like numeric updates:+ // each is applied and the last one wins (this is not a multi-valued update).+ writer.updateDocValues(+ new Term("id", "doc-0"),+ new SortedNumericDocValuesField("val", 1),+ new SortedNumericDocValuesField("val", 2));+ try (DirectoryReader reader = DirectoryReader.open(writer)) {+ assertArrayEquals(new long[] {2}, collect(reader).get("doc-0"));+ }+ IOUtils.close(writer, dir);+ }++ public void testUpdateOverMultiValuedBase() throws Exception {+ Directory dir = newDirectory();+ IndexWriter writer = new IndexWriter(dir, conf());+ writer.addDocument(doc(0, 1, 2, 3)); // genuinely multi-valued base+ writer.addDocument(doc(1, 7, 8));+ writer.commit();++ // updating doc-0 to a single value succeeds even though the column is multi-valued; doc-1 keeps+ // its whole value set.+ writer.updateSortedNumericDocValue(new Term("id", "doc-0"), "val", 99);+ try (DirectoryReader reader = DirectoryReader.open(writer)) {+ Map<String, long[]> values = collect(reader);+ assertArrayEquals(new long[] {99}, values.get("doc-0"));+ assertArrayEquals(new long[] {7, 8}, values.get("doc-1"));+ }+ IOUtils.close(writer, dir);+ }++ public void testUpdateKeepsOtherDocsMultiValuedAcrossMerge() throws Exception {+ Directory dir = newDirectory();+ IndexWriterConfig conf = conf();+ conf.setMaxBufferedDocs(2);+ IndexWriter writer = new IndexWriter(dir, conf);+ for (int i = 0; i < 6; i++) {+ writer.addDocument(doc(i, i * 10L, i * 10L + 1, i * 10L + 2)); // 3 values each+ if (i % 2 == 1) {+ writer.commit();+ }+ }+ // update a few docs to a single value; the rest must keep their 3-value sets, through a merge.+ writer.updateSortedNumericDocValue(new Term("id", "doc-1"), "val", 500);+ writer.updateSortedNumericDocValue(new Term("id", "doc-4"), "val", 501);+ writer.forceMerge(1);+ try (DirectoryReader reader = DirectoryReader.open(writer)) {+ Map<String, long[]> values = collect(reader);+ for (int i = 0; i < 6; i++) {+ if (i == 1) {+ assertArrayEquals(new long[] {500}, values.get("doc-1"));+ } else if (i == 4) {+ assertArrayEquals(new long[] {501}, values.get("doc-4"));+ } else {+ assertArrayEquals(new long[] {i * 10L, i * 10L + 1, i * 10L + 2}, values.get("doc-" + i));+ }+ }+ }+ IOUtils.close(writer, dir);+ }++ public void testDenseRewriteOverMultiValuedBase() throws Exception {+ // maxDocValuesOverlays == 0 forces a dense full-column rewrite on every update. With a+ // multi-valued base the rewrite merges the base column with the single-valued update per doc+ // (the update winning): the matched doc collapses to the single update value while every other+ // doc keeps its whole (sorted) value set read from the base column.+ try (Directory dir = newDirectory();+ IndexWriter w =+ new IndexWriter(+ dir,+ new IndexWriterConfig(new MockAnalyzer(random())).setMaxDocValuesOverlays(0))) {+ w.addDocument(doc(0, 5, 1, 3)); // multi-valued, unsorted input+ w.addDocument(doc(1, 9, 8));+ w.addDocument(doc(2, 2)); // single-valued+ w.commit();++ w.updateSortedNumericDocValue(new Term("id", "doc-1"), "val", 42);+ try (DirectoryReader reader = DirectoryReader.open(w)) {+ Map<String, long[]> values = collect(reader);+ assertArrayEquals(new long[] {1, 3, 5}, values.get("doc-0")); // untouched, sorted+ assertArrayEquals(new long[] {42}, values.get("doc-1")); // replaced with a single value+ assertArrayEquals(new long[] {2}, values.get("doc-2")); // untouched+ }+ // overlays are disabled, so the update produced a dense column, not a sparse delta.+ SegmentInfos sis = SegmentInfos.readLatestCommit(dir);+ assertTrue(sis.info(0).getDocValuesOverlays().isEmpty());+ TestUtil.checkIndex(dir);+ }+ }++ public void testFoldToDenseOverMultiValuedBase() throws Exception {+ // Drive a fold-to-dense rewrite over a genuinely multi-valued base. With+ // maxDocValuesOverlays==1+ // the first round writes a sparse delta; once its coverage crosses+ // FOLD_TO_DENSE_COVERAGE_RATIO (0.5) the next update folds the field back to a single dense+ // column, merging the single-valued update over the multi-valued base per doc. Docs never+ // updated+ // must keep their full value sets.+ int numDocs = 10;+ try (Directory dir = newDirectory();+ IndexWriter w =+ new IndexWriter(+ dir,+ new IndexWriterConfig(new MockAnalyzer(random()))+ .setMaxDocValuesOverlays(1)+ .setMergePolicy(NoMergePolicy.INSTANCE))) {+ for (int i = 0; i < numDocs; i++) {+ w.addDocument(doc(i, i, i + 100)); // two values each; sorted view is {i, i+100}+ }+ w.commit();++ // Round 1: update 7 of 10 docs (> 50% coverage) so the delta will trigger a fold on the next+ // write. Docs 7, 8, 9 are never touched and stay multi-valued in the base.+ Map<String, long[]> expected = new HashMap<>();+ for (int i = 0; i < numDocs; i++) {+ expected.put("doc-" + i, new long[] {i, i + 100L});+ }+ for (int i = 0; i <= 6; i++) {+ w.updateSortedNumericDocValue(new Term("id", "doc-" + i), "val", 1000L + i);+ expected.put("doc-" + i, new long[] {1000L + i});+ }+ w.commit();++ // Round 2: a single further update now folds to a dense column over the multi-valued base.+ w.updateSortedNumericDocValue(new Term("id", "doc-0"), "val", 2000L);+ expected.put("doc-0", new long[] {2000L});+ w.commit();++ // The fold collapses the field back to one dense column, clearing the sparse overlay.+ SegmentInfos sis = SegmentInfos.readLatestCommit(dir);+ assertEquals(1, sis.size());+ assertTrue(+ "fold-to-dense should have cleared the overlay",+ sis.info(0).getDocValuesOverlays().isEmpty());++ try (DirectoryReader reader = DirectoryReader.open(dir)) {+ Map<String, long[]> values = collect(reader);+ for (Map.Entry<String, long[]> e : expected.entrySet()) {+ assertArrayEquals(e.getKey(), e.getValue(), values.get(e.getKey()));+ }+ }+ TestUtil.checkIndex(dir);+ }+ }++ public void testUpdateThenDelete() throws Exception {+ Directory dir = newDirectory();+ IndexWriter writer = new IndexWriter(dir, conf());+ writer.addDocument(doc(0, 1));+ writer.addDocument(doc(1, 3));+ writer.commit();++ writer.updateSortedNumericDocValue(new Term("id", "doc-0"), "val", 100);+ writer.deleteDocuments(new Term("id", "doc-1"));+ try (DirectoryReader reader = DirectoryReader.open(writer)) {+ Map<String, long[]> values = collect(reader);+ assertArrayEquals(new long[] {100}, values.get("doc-0"));+ assertFalse(values.containsKey("doc-1"));+ }+ IOUtils.close(writer, dir);+ }++ public void testNRTReopenSeesUpdates() throws Exception {+ Directory dir = newDirectory();+ IndexWriter writer = new IndexWriter(dir, conf());+ writer.addDocument(doc(0, 1));+ try (DirectoryReader r1 = DirectoryReader.open(writer)) {+ assertArrayEquals(new long[] {1}, collect(r1).get("doc-0"));+ writer.updateSortedNumericDocValue(new Term("id", "doc-0"), "val", 8);+ try (DirectoryReader r2 = DirectoryReader.openIfChanged(r1)) {+ assertNotNull(r2);+ assertArrayEquals(new long[] {8}, collect(r2).get("doc-0"));+ }+ }+ IOUtils.close(writer, dir);+ }++ public void testStackedUpdatesNewestWins() throws Exception {+ Directory dir = newDirectory();+ IndexWriterConfig conf = conf();+ conf.setMaxDocValuesOverlays(4); // force the overlay path with several stacked deltas+ IndexWriter writer = new IndexWriter(dir, conf);+ writer.addDocument(doc(0, 1));+ writer.addDocument(doc(1, 2));+ writer.commit();+ for (int gen = 0; gen < 5; gen++) {+ writer.updateSortedNumericDocValue(new Term("id", "doc-0"), "val", gen * 10L);+ writer.commit();+ }+ try (DirectoryReader reader = DirectoryReader.open(writer)) {+ assertArrayEquals(new long[] {40}, collect(reader).get("doc-0"));+ assertArrayEquals(new long[] {2}, collect(reader).get("doc-1"));+ }+ IOUtils.close(writer, dir);+ }++ public void testAcrossSegmentsAndMerge() throws Exception {+ Directory dir = newDirectory();+ IndexWriterConfig conf = conf();+ conf.setMaxBufferedDocs(2);+ IndexWriter writer = new IndexWriter(dir, conf);+ for (int i = 0; i < 10; i++) {+ writer.addDocument(doc(i, i));+ if (i % 2 == 1) {+ writer.commit();+ }+ }+ for (int i = 0; i < 10; i++) {+ writer.updateSortedNumericDocValue(new Term("id", "doc-" + i), "val", i * 100L);+ }+ writer.forceMerge(1);+ try (DirectoryReader reader = DirectoryReader.open(writer)) {+ Map<String, long[]> values = collect(reader);+ for (int i = 0; i < 10; i++) {+ assertArrayEquals("doc-" + i, new long[] {i * 100L}, values.get("doc-" + i));+ }+ }+ IOUtils.close(writer, dir);+ }++ public void testRejectWrongType() throws Exception {+ Directory dir = newDirectory();+ IndexWriter writer = new IndexWriter(dir, conf());+ writer.addDocument(doc(0, 1));+ writer.commit();+ // "val" is SORTED_NUMERIC; trying to update it as NUMERIC must fail+ expectThrows(+ IllegalArgumentException.class,+ () -> writer.updateNumericDocValue(new Term("id", "doc-0"), "val", 5L));+ IOUtils.close(writer, dir);+ }++ public void testRejectIndexSortField() throws Exception {+ Directory dir = newDirectory();+ IndexWriterConfig conf = conf();+ conf.setIndexSort(new Sort(new SortedNumericSortField("val", SortField.Type.LONG)));+ IndexWriter writer = new IndexWriter(dir, conf);+ writer.addDocument(doc(0, 1));+ writer.commit();+ expectThrows(+ IllegalArgumentException.class,+ () -> writer.updateSortedNumericDocValue(new Term("id", "doc-0"), "val", 5L));+ IOUtils.close(writer, dir);+ }++ public void testFoldsToDenseOnFullCorpusUpdates() throws Exception {+ // Mirrors TestIncrementalDocValuesUpdates#testFoldsToDenseOnFullCorpusUpdates for the+ // single-valued sorted-numeric column, which reuses the numeric overlay/fold machinery: once+ // the+ // stacked deltas cover the whole corpus the field folds back to a single dense column.+ try (Directory dir = newDirectory();+ IndexWriter w =+ new IndexWriter(+ dir,+ new IndexWriterConfig(new MockAnalyzer(random())).setMaxDocValuesOverlays(2))) {+ int numDocs = 40;+ for (int i = 0; i < numDocs; i++) {+ w.addDocument(doc(i, i));+ }+ long expected = 0;+ for (int round = 0; round < 12; round++) {+ expected = 1000L + round;+ for (int i = 0; i < numDocs; i++) {+ w.updateSortedNumericDocValue(new Term("id", "doc-" + i), "val", expected + i);+ }+ w.commit();+ }+ try (DirectoryReader reader = DirectoryReader.open(w)) {+ Map<String, long[]> values = collect(reader);+ for (int i = 0; i < numDocs; i++) {+ assertArrayEquals(new long[] {expected + i}, values.get("doc-" + i));+ }+ }+ TestUtil.checkIndex(dir);+ }+ }++ public void testSparseFoldOverDenseBase() throws Exception {+ // Mirrors TestIncrementalDocValuesUpdates#testSparseFoldOverDenseBase: after a fold-to-dense+ // the+ // field's base becomes a dense generation (!= -1), and further updates must keep stacking a+ // sparse overlay over that dense base rather than clearing it. The singleton wrap/unwrap of the+ // sorted-numeric fold must preserve this.+ int numDocs = 10;+ try (Directory dir = newDirectory();+ IndexWriter w =+ new IndexWriter(+ dir,+ new IndexWriterConfig(new MockAnalyzer(random()))+ .setMaxDocValuesOverlays(1)+ .setMergePolicy(NoMergePolicy.INSTANCE))) {+ for (int i = 0; i < numDocs; i++) {+ w.addDocument(doc(i, i));+ }+ w.commit();+ // Update every doc twice: the second round's coverage reaches maxDoc and folds to a dense+ // column, so the field's base becomes a dense generation rather than the core column.+ for (int round = 1; round <= 2; round++) {+ for (int i = 0; i < numDocs; i++) {+ w.updateSortedNumericDocValue(new Term("id", "doc-" + i), "val", 100L * round + i);+ }+ w.commit();+ }+ // Now keep updating a single doc across two more folds over the dense base.+ for (int round = 0; round < 2; round++) {+ w.updateSortedNumericDocValue(new Term("id", "doc-0"), "val", 999L + round);+ w.commit();+ }+ // The last fold over the dense base must stay sparse: an overlay whose base is a dense+ // generation (!= -1) with a folded delta, not a full-column rewrite that clears the overlay.+ SegmentInfos sis = SegmentInfos.readLatestCommit(dir);+ assertEquals(1, sis.size());+ Map<Integer, long[]> overlays = sis.info(0).getDocValuesOverlays();+ assertFalse("expected a sparse overlay over the dense base", overlays.isEmpty());+ long[] packed = overlays.values().iterator().next();+ assertTrue("base should be a dense generation, not the core column", packed[0] != -1);+ assertTrue("expected at least one delta over the dense base", packed.length >= 2);+ try (DirectoryReader reader = DirectoryReader.open(dir)) {+ Map<String, long[]> values = collect(reader);+ assertArrayEquals(new long[] {1000L}, values.get("doc-0"));+ for (int i = 1; i < numDocs; i++) {+ assertArrayEquals(new long[] {200L + i}, values.get("doc-" + i));+ }+ }+ TestUtil.checkIndex(dir);+ }+ }++ public void testRandom() throws Exception {+ Directory dir = newDirectory();+ IndexWriter writer = new IndexWriter(dir, conf());+ int numDocs = atLeast(50);+ Map<String, long[]> expected = new HashMap<>();+ for (int i = 0; i < numDocs; i++) {+ // start single-valued so every doc can be updated in place+ long v = random().nextInt(1000);+ writer.addDocument(doc(i, v));+ expected.put("doc-" + i, new long[] {v});+ if (rarely()) {+ writer.commit();+ }+ }+ int numUpdates = atLeast(100);+ for (int u = 0; u < numUpdates; u++) {+ int id = random().nextInt(numDocs);+ long v = random().nextInt(1000);+ writer.updateSortedNumericDocValue(new Term("id", "doc-" + id), "val", v);+ expected.put("doc-" + id, new long[] {v});+ if (rarely()) {+ writer.commit();+ }+ if (rarely()) {+ writer.forceMerge(1 + random().nextInt(3));+ }+ }+ try (DirectoryReader reader = DirectoryReader.open(writer)) {+ Map<String, long[]> actual = collect(reader);+ for (Map.Entry<String, long[]> e : expected.entrySet()) {+ assertArrayEquals(e.getKey(), e.getValue(), actual.get(e.getKey()));+ }+ }+ IOUtils.close(writer, dir);+ }++ public void testRandomMultiValued() throws Exception {+ Directory dir = newDirectory();+ IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));+ // Low overlay budget (0..2) so stacked deltas frequently fold to a dense rewrite over the+ // multi-valued base, exercising the base+update dense merge against the model.+ conf.setMaxDocValuesOverlays(random().nextInt(3));+ IndexWriter writer = new IndexWriter(dir, conf);+ int numDocs = atLeast(50);+ Map<String, long[]> expected = new HashMap<>();+ for (int i = 0; i < numDocs; i++) {+ // Seed genuinely multi-valued docs (1..4 values). Many docs are never updated below, so they+ // keep their multi-valued sets and force the dense rewrites to read a multi-valued base.+ long[] vals = new long[1 + random().nextInt(4)];+ for (int j = 0; j < vals.length; j++) {+ vals[j] = random().nextInt(1000);+ }+ writer.addDocument(doc(i, vals));+ long[] sorted = vals.clone();+ Arrays.sort(sorted); // a sorted-numeric column returns its values ascending, keeping dups+ expected.put("doc-" + i, sorted);+ if (rarely()) {+ writer.commit();+ }+ }+ int numUpdates = atLeast(100);+ for (int u = 0; u < numUpdates; u++) {+ // Only update a subset of docs so a good fraction stay multi-valued in the base.+ int id = random().nextInt(numDocs);+ long v = random().nextInt(1000);+ writer.updateSortedNumericDocValue(new Term("id", "doc-" + id), "val", v);+ expected.put("doc-" + id, new long[] {v}); // an update replaces the whole set with one value+ if (rarely()) {+ writer.commit();+ }+ if (rarely()) {+ writer.forceMerge(1 + random().nextInt(3));+ }+ }+ try (DirectoryReader reader = DirectoryReader.open(writer)) {+ Map<String, long[]> actual = collect(reader);+ for (Map.Entry<String, long[]> e : expected.entrySet()) {+ assertArrayEquals(e.getKey(), e.getValue(), actual.get(e.getKey()));+ }+ }+ IOUtils.close(writer, dir);+ }++ public void testMixedSingleAndMultiValuedSegments() throws Exception {+ Directory dir = newDirectory();+ IndexWriter writer = new IndexWriter(dir, conf());++ // Alternate segments: even segments hold only single-valued "val" docs (a singleton column on+ // disk), odd segments hold multi-valued "val" docs (a true sorted-numeric column). Every doc+ // also has an always-single-valued "sv" field. Merging then combines singleton and+ // non-singleton columns for "val", while "sv" stays single-valued everywhere.+ int numSegments = 6;+ int docsPerSegment = 5;+ Map<String, long[]> expectedVal = new HashMap<>();+ Map<String, long[]> expectedSv = new HashMap<>();+ int id = 0;+ for (int s = 0; s < numSegments; s++) {+ boolean multiValued = (s % 2 == 1);+ for (int d = 0; d < docsPerSegment; d++, id++) {+ Document doc = new Document();+ doc.add(new StringField("id", "doc-" + id, Store.NO));+ long sv = id * 7L;+ doc.add(new SortedNumericDocValuesField("sv", sv));+ expectedSv.put("doc-" + id, new long[] {sv});+ long[] vals = multiValued ? new long[] {id, id + 1000L} : new long[] {id};+ for (long v : vals) {+ doc.add(new SortedNumericDocValuesField("val", v));+ }+ long[] sorted = vals.clone();+ Arrays.sort(sorted);+ expectedVal.put("doc-" + id, sorted);+ writer.addDocument(doc);+ }+ writer.commit(); // one segment per iteration+ }+ int totalDocs = id;++ // Update every third doc's "val" (a single-valued update: collapses a multi-valued doc, or just+ // changes a single-valued one) and its "sv" (which stays single-valued).+ for (int u = 0; u < totalDocs; u += 3) {+ writer.updateSortedNumericDocValue(new Term("id", "doc-" + u), "val", 5000L + u);+ expectedVal.put("doc-" + u, new long[] {5000L + u});+ writer.updateSortedNumericDocValue(new Term("id", "doc-" + u), "sv", 9000L + u);+ expectedSv.put("doc-" + u, new long[] {9000L + u});+ }++ writer.forceMerge(1);++ try (DirectoryReader reader = DirectoryReader.open(writer)) {+ assertEquals(1, reader.leaves().size());+ LeafReader leaf = reader.leaves().get(0).reader();+ // "val" mixes multi-valued docs from the odd segments, so its merged column is not a+ // singleton.+ assertFalse(+ "val column should be multi-valued",+ DocValues.isSingleton(leaf.getSortedNumericDocValues("val")));+ // "sv" was single-valued on every doc and only ever updated with a single value, so its+ // merged column must be a singleton.+ assertTrue(+ "sv column should be a singleton",+ DocValues.isSingleton(leaf.getSortedNumericDocValues("sv")));++ Map<String, long[]> actualVal = collect(reader, "val");+ Map<String, long[]> actualSv = collect(reader, "sv");+ for (int i = 0; i < totalDocs; i++) {+ String key = "doc-" + i;+ assertArrayEquals(key + " val", expectedVal.get(key), actualVal.get(key));+ assertArrayEquals(key + " sv", expectedSv.get(key), actualSv.get(key));+ }+ }+ writer.commit(); // persist the merged segment + updates so checkIndex validates them+ TestUtil.checkIndex(dir);+ IOUtils.close(writer, dir);+ }++ public void testUpdateAllToSingleValueBecomesSingletonAfterMerge() throws Exception {+ Directory dir = newDirectory();+ // Overlays on, so the single-valued updates are written as sparse deltas over the multi-valued+ // base rather than as a dense rewrite.+ IndexWriterConfig conf =+ newIndexWriterConfig(new MockAnalyzer(random()))+ .setMaxDocValuesOverlays(1 + random().nextInt(4));+ IndexWriter writer = new IndexWriter(dir, conf);+ int numDocs = 20;+ for (int i = 0; i < numDocs; i++) {+ writer.addDocument(doc(i, i, i + 100L, i + 200L)); // 3 values each -> multi-valued column+ }+ writer.commit();++ // Update every doc to a single value.+ Map<String, long[]> expected = new HashMap<>();+ for (int i = 0; i < numDocs; i++) {+ writer.updateSortedNumericDocValue(new Term("id", "doc-" + i), "val", 1000L + i);+ expected.put("doc-" + i, new long[] {1000L + i});+ }++ // Add one more (single-valued) doc in its own segment: this guarantees at least two segments,+ // so forceMerge(1) actually runs a merge (a forceMerge of a single segment is a no-op and would+ // leave the sparse overlay in place, which reports non-singleton because its base layer is+ // multi-valued). The merge materializes everything into one dense column; every doc now holds a+ // single value, so the codec stores it without an addresses table and it reads back as a+ // singleton.+ writer.addDocument(doc(numDocs, 5000L));+ expected.put("doc-" + numDocs, new long[] {5000L});+ writer.commit();+ writer.forceMerge(1);++ try (DirectoryReader reader = DirectoryReader.open(writer)) {+ assertEquals(1, reader.leaves().size());+ LeafReader leaf = reader.leaves().get(0).reader();+ assertTrue(+ "a merged column whose every doc holds a single value should be a singleton",+ DocValues.isSingleton(leaf.getSortedNumericDocValues("val")));+ Map<String, long[]> actual = collect(reader);+ for (int i = 0; i <= numDocs; i++) {+ assertArrayEquals("doc-" + i, expected.get("doc-" + i), actual.get("doc-" + i));+ }+ }+ writer.commit();+ TestUtil.checkIndex(dir);+ IOUtils.close(writer, dir);+ }++ /** Collect field "val" for all live docs, keyed by the "id" stored term. */+ private static Map<String, long[]> collect(DirectoryReader reader) throws Exception {+ return collect(reader, "val");+ }++ /** Collect the given sorted-numeric field for all live docs, keyed by the "id" stored term. */+ private static Map<String, long[]> collect(DirectoryReader reader, String field)+ throws Exception {+ Map<String, long[]> result = new TreeMap<>();+ for (LeafReaderContext ctx : reader.leaves()) {+ LeafReader lr = ctx.reader();+ SortedNumericDocValues dv = lr.getSortedNumericDocValues(field);+ Terms idTerms = lr.terms("id");+ if (idTerms == null || dv == null) {+ continue;+ }+ Map<Integer, String> idByDoc = new HashMap<>();+ TermsEnum te = idTerms.iterator();+ org.apache.lucene.util.BytesRef term;+ while ((term = te.next()) != null) {+ PostingsEnum pe = te.postings(null, PostingsEnum.NONE);+ int d;+ while ((d = pe.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {+ idByDoc.put(d, term.utf8ToString());+ }+ }+ Bits liveDocs = lr.getLiveDocs();+ int doc;+ while ((doc = dv.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {+ if (liveDocs != null && liveDocs.get(doc) == false) {+ continue;+ }+ int count = dv.docValueCount();+ long[] arr = new long[count];+ for (int i = 0; i < count; i++) {+ arr[i] = dv.nextValue();+ }+ result.put(idByDoc.get(doc), arr);+ }+ }+ return result;+ }+}