Skip to content

Apache Lucene open-source search software

Allow single-valued updates for SortedNumericDocValues (#16581)

Houston Putman169cbffd12f4
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 16a6728248c..f8a74b323ff 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -407,6 +407,12 @@ New Features
it is not possible to re-encode to a different quantization width or center vectors again.
(Trevor McCulloch)
+* GITHUB#16580: Allow in-place updates of SORTED_NUMERIC doc-values via
+ IndexWriter#updateSortedNumericDocValue (and IndexWriter#updateDocValues with a single
+ SortedNumericDocValuesField). An update sets a single value per matched document; the field may be
+ single- or multi-valued. Single-valued columns reuse the numeric update path (including the
+ incremental delta overlay), and multi-valued columns are merged per-doc. (Houston Putman)
+
Improvements
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValues.java b/lucene/core/src/java/org/apache/lucene/index/DocValues.java
index c51d29b7d6f..e01dd6e5bfe 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocValues.java
@@ -199,6 +199,14 @@ public final class DocValues {
return new SingletonSortedNumericDocValues(dv);
}
+ /**
+ * Returns whether the provided SortedNumericDocValues is a single-valued view of a
+ * NumericDocValues.
+ */
+ static boolean isSingleton(SortedNumericDocValues dv) {
+ return dv instanceof SingletonSortedNumericDocValues;
+ }
+
// some helpers, for transition from fieldcache apis.
// as opposed to the LeafReader apis (which must be strict for consistency), these are lenient
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java
index 9be34e997b2..f84ffd0b817 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java
@@ -153,21 +153,38 @@ abstract class DocValuesUpdate {
}
}
- /** An in-place update to a numeric DocValues field */
+ /**
+ * An in-place update to a numeric DocValues field.
+ *
+ * <p>Also used for {@link org.apache.lucene.document.SortedNumericDocValuesField} updates ({@code
+ * type == SORTED_NUMERIC}): such an update sets a single value per matched document, so its
+ * payload is identical to a numeric update — only the doc-values type differs, which drives the
+ * singleton wrapping at write/read time. The updated field may itself be single- or multi-valued:
+ * a single-valued column is physically stored as a {@link NumericDocValues} column (the codec
+ * only writes an addresses table when a document has more than one value) and reuses the numeric
+ * write path, while a multi-valued column is merged per-doc.
+ */
static final class NumericDocValuesUpdate extends DocValuesUpdate {
private final long value;
NumericDocValuesUpdate(Term term, String field, long value) {
- this(term, field, value, BufferedUpdates.MAX_INT, true);
+ this(DocValuesType.NUMERIC, term, field, value, BufferedUpdates.MAX_INT, true);
}
NumericDocValuesUpdate(Term term, String field, Long value) {
- this(term, field, value != null ? value : -1, BufferedUpdates.MAX_INT, value != null);
+ this(DocValuesType.NUMERIC, term, field, value);
+ }
+
+ /** Creates a numeric or sorted-numeric update, depending on {@code type}. */
+ NumericDocValuesUpdate(DocValuesType type, Term term, String field, Long value) {
+ this(type, term, field, value != null ? value : -1, BufferedUpdates.MAX_INT, value != null);
}
private NumericDocValuesUpdate(
- Term term, String field, long value, int docIDUpTo, boolean hasValue) {
- super(DocValuesType.NUMERIC, term, field, docIDUpTo, hasValue);
+ DocValuesType type, Term term, String field, long value, int docIDUpTo, boolean hasValue) {
+ super(type, term, field, docIDUpTo, hasValue);
+ assert type == DocValuesType.NUMERIC || type == DocValuesType.SORTED_NUMERIC
+ : "unexpected type: " + type;
this.value = value;
}
@@ -175,7 +192,7 @@ abstract class DocValuesUpdate {
if (docIDUpTo == this.docIDUpTo) {
return this;
}
- return new NumericDocValuesUpdate(term, field, value, docIDUpTo, hasValue);
+ return new NumericDocValuesUpdate(type, term, field, value, docIDUpTo, hasValue);
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java
index 78e913ce9d1..d79864a62ca 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterDeleteQueue.java
@@ -501,6 +501,9 @@ final class DocumentsWriterDeleteQueue implements Accountable, Closeable {
for (DocValuesUpdate update : item) {
switch (update.type) {
case NUMERIC:
+ case SORTED_NUMERIC:
+ // a sorted-numeric update carries a single value, so it shares the numeric update
+ // object and buffer
bufferedUpdates.addNumericUpdate((NumericDocValuesUpdate) update, docIDUpto);
break;
case BINARY:
@@ -509,7 +512,6 @@ final class DocumentsWriterDeleteQueue implements Accountable, Closeable {
case NONE:
case SORTED:
case SORTED_SET:
- case SORTED_NUMERIC:
default:
throw new IllegalArgumentException(
update.type + " DocValues updates not supported yet!");
diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldUpdatesBuffer.java b/lucene/core/src/java/org/apache/lucene/index/FieldUpdatesBuffer.java
index a4d38dcd203..6408988d06a 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FieldUpdatesBuffer.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FieldUpdatesBuffer.java
@@ -61,11 +61,15 @@ final class FieldUpdatesBuffer {
private long maxNumeric = Long.MIN_VALUE;
private long minNumeric = Long.MAX_VALUE;
private String[] fields;
+ // the actual doc-values type of the updated field. A SORTED_NUMERIC update sets a single long
+ // value per doc, identical to a numeric update, so it shares the numeric buffering path but keeps
+ // its own type so the write/read side can apply the singleton wrapping.
+ private final DocValuesType type;
private final boolean isNumeric;
private boolean finished = false;
private FieldUpdatesBuffer(
- Counter bytesUsed, DocValuesUpdate initialValue, int docUpTo, boolean isNumeric) {
+ Counter bytesUsed, DocValuesUpdate initialValue, int docUpTo, DocValuesType type) {
this.bytesUsed = bytesUsed;
this.bytesUsed.addAndGet(SELF_SHALLOW_SIZE);
termValues = new BytesRefArray(bytesUsed);
@@ -77,7 +81,9 @@ final class FieldUpdatesBuffer {
hasValues = new FixedBitSet(1);
bytesUsed.addAndGet(hasValues.ramBytesUsed());
}
- this.isNumeric = isNumeric;
+ this.type = type;
+ // NUMERIC and SORTED_NUMERIC updates both carry a single long, so they share the storage path.
+ this.isNumeric = type == DocValuesType.NUMERIC || type == DocValuesType.SORTED_NUMERIC;
byteValues = isNumeric ? null : new BytesRefArray(bytesUsed);
}
@@ -87,7 +93,8 @@ final class FieldUpdatesBuffer {
FieldUpdatesBuffer(
Counter bytesUsed, DocValuesUpdate.NumericDocValuesUpdate initialValue, int docUpTo) {
- this(bytesUsed, initialValue, docUpTo, true);
+ // NUMERIC and SORTED_NUMERIC share this buffer; the update carries which one.
+ this(bytesUsed, initialValue, docUpTo, initialValue.type);
if (initialValue.hasValue()) {
numericValues = new long[] {initialValue.getValue()};
maxNumeric = minNumeric = initialValue.getValue();
@@ -99,12 +106,16 @@ final class FieldUpdatesBuffer {
FieldUpdatesBuffer(
Counter bytesUsed, DocValuesUpdate.BinaryDocValuesUpdate initialValue, int docUpTo) {
- this(bytesUsed, initialValue, docUpTo, false);
+ this(bytesUsed, initialValue, docUpTo, DocValuesType.BINARY);
if (initialValue.hasValue()) {
byteValues.append(initialValue.getValue());
}
}
+ DocValuesType getType() {
+ return type;
+ }
+
long getMaxNumeric() {
assert isNumeric;
if (minNumeric == Long.MAX_VALUE && maxNumeric == Long.MIN_VALUE) {
diff --git a/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java b/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java
index 13e6db358ca..4ffb200facb 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java
@@ -284,15 +284,21 @@ final class FrozenBufferedUpdates {
}
if (dvUpdates == null) {
if (isNumeric) {
+ DocValuesType dvType = value.getType();
if (value.hasSingleValue()) {
dvUpdates =
new NumericDocValuesFieldUpdates.SingleValueNumericDocValuesFieldUpdates(
- delGen, updateField, segState.reader.maxDoc(), value.getNumericValue(0));
+ delGen,
+ updateField,
+ dvType,
+ segState.reader.maxDoc(),
+ value.getNumericValue(0));
} else {
dvUpdates =
new NumericDocValuesFieldUpdates(
delGen,
updateField,
+ dvType,
value.getMinNumeric(),
value.getMaxNumeric(),
segState.reader.maxDoc());
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index 62204c11d4b..45b59fa46dd 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -1736,14 +1736,14 @@ public class IndexWriter
k -> {
switch (update.type) {
case NUMERIC:
+ case SORTED_NUMERIC:
return new NumericDocValuesFieldUpdates(
- nextGen, k, rld.info.info.maxDoc());
+ nextGen, k, update.type, rld.info.info.maxDoc());
case BINARY:
return new BinaryDocValuesFieldUpdates(
nextGen, k, rld.info.info.maxDoc());
case NONE:
case SORTED:
- case SORTED_NUMERIC:
case SORTED_SET:
default:
throw new AssertionError("type: " + update.type + " is not supported");
@@ -1752,6 +1752,7 @@ public class IndexWriter
if (update.hasValue()) {
switch (update.type) {
case NUMERIC:
+ case SORTED_NUMERIC:
docValuesFieldUpdates.add(
leafDocId, ((NumericDocValuesUpdate) update).getValue());
break;
@@ -1762,7 +1763,6 @@ public class IndexWriter
case NONE:
case SORTED:
case SORTED_SET:
- case SORTED_NUMERIC:
default:
throw new AssertionError("type: " + update.type + " is not supported");
}
@@ -2017,6 +2017,45 @@ public class IndexWriter
}
}
+ /**
+ * Updates a document's {@link org.apache.lucene.index.SortedNumericDocValues} for <code>field
+ * </code> to the given <code>value</code>. You can only update fields that already exist in the
+ * index, not add new fields through this method. You can only update fields that were indexed
+ * only with doc values.
+ *
+ * <p><b>NOTE:</b> the update sets a single value per matched document, replacing all values that
+ * document previously had. The field itself may be single- or multi-valued; documents not matched
+ * by <code>term</code> keep their existing (possibly multiple) values.
+ *
+ * @param term the term to identify the document(s) to be updated
+ * @param field field name of the {@link org.apache.lucene.index.SortedNumericDocValues} field
+ * @param value new value for the field
+ * @return The <a href="#sequence_number">sequence number</a> for this operation
+ * @throws CorruptIndexException if the index is corrupt
+ * @throws IOException if there is a low-level IO error
+ */
+ public long updateSortedNumericDocValue(Term term, String field, long value) throws IOException {
+ ensureOpen();
+ // Checked before the doc-values-type check below, mirroring updateNumericDocValue/
+ // updateBinaryDocValue, so an index-sort field surfaces this clear message.
+ if (config.getIndexSortFields().contains(field)) {
+ throw new IllegalArgumentException(
+ "cannot update docvalues field involved in the index sort, field="
+ + field
+ + ", sort="
+ + config.getIndexSort());
+ }
+ globalFieldNumberMap.verifyOrCreateDvOnlyField(field, DocValuesType.SORTED_NUMERIC, true);
+ try {
+ return maybeProcessEvents(
+ docWriter.updateDocValues(
+ new NumericDocValuesUpdate(DocValuesType.SORTED_NUMERIC, term, field, value)));
+ } catch (Error tragedy) {
+ tragicEvent(tragedy, "updateSortedNumericDocValue");
+ throw tragedy;
+ }
+ }
+
/**
* Updates documents' DocValues fields to the given values. Each field update is applied to the
* set of documents that are associated with the {@link Term} to the same value. All updates are
@@ -2066,19 +2105,22 @@ public class IndexWriter
switch (dvType) {
case NUMERIC:
+ case SORTED_NUMERIC:
Long value = (Long) f.numericValue();
- dvUpdates[i] = new NumericDocValuesUpdate(term, f.name(), value);
+ dvUpdates[i] = new NumericDocValuesUpdate(dvType, term, f.name(), value);
break;
case BINARY:
dvUpdates[i] = new BinaryDocValuesUpdate(term, f.name(), f.binaryValue());
break;
case NONE:
case SORTED:
- case SORTED_NUMERIC:
case SORTED_SET:
default:
throw new IllegalArgumentException(
- "can only update NUMERIC or BINARY fields: field=" + f.name() + ", type=" + dvType);
+ "can only update NUMERIC, BINARY or SORTED_NUMERIC fields: field="
+ + f.name()
+ + ", type="
+ + dvType);
}
}
return dvUpdates;
@@ -4514,9 +4556,6 @@ public class IndexWriter
return mergedDeletesAndUpdates;
}
- /** Sentinel for "no doc-values value" (a reset, or a doc that simply has no value). */
- private static final Object DV_NO_VALUE = new Object();
-
/**
* Builds the doc-values-update merge carry-over from the source segments' on-disk state plus
* their residual (resolved but not-yet-written) pending updates, remapping to merged docIDs.
@@ -4561,7 +4600,9 @@ public class IndexWriter
FieldInfos baseFieldInfos = baseline.getFieldInfos();
for (FieldInfo fi : current.getFieldInfos()) {
DocValuesType t = fi.getDocValuesType();
- if (t != DocValuesType.NUMERIC && t != DocValuesType.BINARY) {
+ if (t != DocValuesType.NUMERIC
+ && t != DocValuesType.BINARY
+ && t != DocValuesType.SORTED_NUMERIC) {
continue;
}
FieldInfo baseFi = baseFieldInfos.fieldInfo(fi.name);
@@ -4609,7 +4650,7 @@ public class IndexWriter
p =
u.type == DocValuesType.BINARY
? new BinaryDocValuesFieldUpdates(u.delGen, u.field, mergedMaxDoc)
- : new NumericDocValuesFieldUpdates(u.delGen, u.field, mergedMaxDoc);
+ : new NumericDocValuesFieldUpdates(u.delGen, u.field, u.type, mergedMaxDoc);
byGen.put(u.delGen, p);
}
if (it.hasValue()) {
@@ -4639,7 +4680,8 @@ public class IndexWriter
p =
fi.getDocValuesType() == DocValuesType.BINARY
? new BinaryDocValuesFieldUpdates(delGen, fi.name, mergedMaxDoc)
- : new NumericDocValuesFieldUpdates(delGen, fi.name, mergedMaxDoc);
+ : new NumericDocValuesFieldUpdates(
+ delGen, fi.name, fi.getDocValuesType(), mergedMaxDoc);
byGen.put(delGen, p);
}
return p;
@@ -4659,51 +4701,130 @@ public class IndexWriter
int mergedMaxDoc,
LongObjectHashMap<DocValuesFieldUpdates> byGen)
throws IOException {
+ switch (fi.getDocValuesType()) {
+ case NUMERIC ->
+ addNumericDiskDiffToPacket(
+ fi, baseline, current, segDocMap, diskDelGen, mergedMaxDoc, byGen);
+ case BINARY ->
+ addBinaryDiskDiffToPacket(
+ fi, baseline, current, segDocMap, diskDelGen, mergedMaxDoc, byGen);
+ case SORTED_NUMERIC ->
+ addSortedNumericDiskDiffToPacket(
+ fi, baseline, current, segDocMap, diskDelGen, mergedMaxDoc, byGen);
+ // $CASES-OMITTED$
+ default ->
+ throw new AssertionError(
+ "unexpected doc-values type for update carry-over: " + fi.getDocValuesType());
+ }
+ }
+
+ private static void addNumericDiskDiffToPacket(
+ FieldInfo fi,
+ CodecReader baseline,
+ CodecReader current,
+ MergeState.DocMap segDocMap,
+ long diskDelGen,
+ int mergedMaxDoc,
+ LongObjectHashMap<DocValuesFieldUpdates> byGen)
+ throws IOException {
final int maxDoc = current.maxDoc();
- final boolean binary = fi.getDocValuesType() == DocValuesType.BINARY;
- BinaryDocValues curB = binary ? current.getBinaryDocValues(fi.name) : null;
- BinaryDocValues baseB = binary ? baseline.getBinaryDocValues(fi.name) : null;
- NumericDocValues curN = binary ? null : current.getNumericDocValues(fi.name);
- NumericDocValues baseN = binary ? null : baseline.getNumericDocValues(fi.name);
+ NumericDocValues cur = current.getNumericDocValues(fi.name);
+ NumericDocValues base = baseline.getNumericDocValues(fi.name);
for (int doc = 0; doc < maxDoc; doc++) {
int md = segDocMap.get(doc);
if (md == -1) {
continue;
}
- Object curVal;
- Object baseVal;
- if (binary) {
- // Live BytesRefs suffice to compare; only a changed value is copied, below, when it is
- // stored.
- curVal = (curB != null && curB.advanceExact(doc)) ? curB.binaryValue() : DV_NO_VALUE;
- baseVal = (baseB != null && baseB.advanceExact(doc)) ? baseB.binaryValue() : DV_NO_VALUE;
+ boolean hasCur = cur != null && cur.advanceExact(doc);
+ boolean hasBase = base != null && base.advanceExact(doc);
+ long curVal = hasCur ? cur.longValue() : 0;
+ if (hasCur == hasBase && (hasCur == false || curVal == base.longValue())) {
+ continue; // unchanged: both absent, or both present with an equal value
+ }
+ DocValuesFieldUpdates p = getOrCreatePacket(byGen, diskDelGen, fi, mergedMaxDoc);
+ if (hasCur == false) {
+ p.reset(md);
} else {
- curVal =
- (curN != null && curN.advanceExact(doc)) ? Long.valueOf(curN.longValue()) : DV_NO_VALUE;
- baseVal =
- (baseN != null && baseN.advanceExact(doc))
- ? Long.valueOf(baseN.longValue())
- : DV_NO_VALUE;
- }
- if (dvEquals(curVal, baseVal)) {
+ p.add(md, curVal);
+ }
+ }
+ }
+
+ private static void addBinaryDiskDiffToPacket(
+ FieldInfo fi,
+ CodecReader baseline,
+ CodecReader current,
+ MergeState.DocMap segDocMap,
+ long diskDelGen,
+ int mergedMaxDoc,
+ LongObjectHashMap<DocValuesFieldUpdates> byGen)
+ throws IOException {
+ final int maxDoc = current.maxDoc();
+ BinaryDocValues cur = current.getBinaryDocValues(fi.name);
+ BinaryDocValues base = baseline.getBinaryDocValues(fi.name);
+ for (int doc = 0; doc < maxDoc; doc++) {
+ int md = segDocMap.get(doc);
+ if (md == -1) {
continue;
}
+ // cur and base are independent readers, so their live BytesRefs can be compared directly; a
+ // changed value is deep-copied only when it is stored below.
+ boolean hasCur = cur != null && cur.advanceExact(doc);
+ BytesRef curVal = hasCur ? cur.binaryValue() : null;
+ boolean hasBase = base != null && base.advanceExact(doc);
+ if (hasCur == hasBase && (hasCur == false || curVal.bytesEquals(base.binaryValue()))) {
+ continue; // unchanged: both absent, or both present with an equal value
+ }
DocValuesFieldUpdates p = getOrCreatePacket(byGen, diskDelGen, fi, mergedMaxDoc);
- if (curVal == DV_NO_VALUE) {
+ if (hasCur == false) {
p.reset(md);
- } else if (binary) {
- ((BinaryDocValuesFieldUpdates) p).add(md, BytesRef.deepCopyOf((BytesRef) curVal));
} else {
- ((NumericDocValuesFieldUpdates) p).add(md, (Long) curVal);
+ p.add(md, BytesRef.deepCopyOf(curVal));
}
}
}
- private static boolean dvEquals(Object a, Object b) {
- if (a == DV_NO_VALUE || b == DV_NO_VALUE) {
- return a == b;
+ /**
+ * The sorted-numeric analog: a doc whose whole value set changed during the merge was updated,
+ * and an update always sets a single value (an unchanged multi-valued doc compares equal and is
+ * skipped), so a changed doc is emitted as either a single value or a reset.
+ */
+ private static void addSortedNumericDiskDiffToPacket(
+ FieldInfo fi,
+ CodecReader baseline,
+ CodecReader current,
+ MergeState.DocMap segDocMap,
+ long diskDelGen,
+ int mergedMaxDoc,
+ LongObjectHashMap<DocValuesFieldUpdates> byGen)
+ throws IOException {
+ final int maxDoc = current.maxDoc();
+ SortedNumericDocValues cur = current.getSortedNumericDocValues(fi.name);
+ SortedNumericDocValues base = baseline.getSortedNumericDocValues(fi.name);
+ for (int doc = 0; doc < maxDoc; doc++) {
+ int md = segDocMap.get(doc);
+ if (md == -1) {
+ continue;
+ }
+ boolean hasCur = cur != null && cur.advanceExact(doc);
+ if (hasCur && cur.docValueCount() > 1) {
+ // Updates are always single-valued, so a still-multi-valued doc cannot have been updated.
+ continue;
+ }
+ long curVal = hasCur ? cur.nextValue() : 0;
+ boolean hasBase = base != null && base.advanceExact(doc);
+ if (hasCur == false && hasBase == false) {
+ continue;
+ } else if (hasCur && hasBase && base.docValueCount() == 1 && base.nextValue() == curVal) {
+ continue;
+ }
+ DocValuesFieldUpdates p = getOrCreatePacket(byGen, diskDelGen, fi, mergedMaxDoc);
+ if (hasCur == false) {
+ p.reset(md);
+ } else {
+ p.add(md, curVal);
+ }
}
- return a.equals(b);
}
/**
diff --git a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
index 41b642ddf59..edeec613ff0 100644
--- a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
@@ -67,7 +67,11 @@ final class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
private final long minValue;
NumericDocValuesFieldUpdates(long delGen, String field, int maxDoc) {
- super(maxDoc, delGen, field, DocValuesType.NUMERIC);
+ this(delGen, field, DocValuesType.NUMERIC, maxDoc);
+ }
+
+ NumericDocValuesFieldUpdates(long delGen, String field, DocValuesType type, int maxDoc) {
+ super(maxDoc, delGen, field, type);
// we don't know the min/max range so we use the growable writer here to adjust as we go.
values = new PagedGrowableWriter(1, PAGE_SIZE, 1, PackedInts.DEFAULT);
minValue = 0;
@@ -75,7 +79,12 @@ final class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
NumericDocValuesFieldUpdates(
long delGen, String field, long minValue, long maxValue, int maxDoc) {
- super(maxDoc, delGen, field, DocValuesType.NUMERIC);
+ this(delGen, field, DocValuesType.NUMERIC, minValue, maxValue, maxDoc);
+ }
+
+ NumericDocValuesFieldUpdates(
+ long delGen, String field, DocValuesType type, long minValue, long maxValue, int maxDoc) {
+ super(maxDoc, delGen, field, type);
assert minValue <= maxValue
: "minValue must be <= maxValue [" + minValue + " > " + maxValue + "]";
int bitsPerValue = PackedInts.unsignedBitsRequired(maxValue - minValue);
@@ -141,7 +150,12 @@ final class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
private boolean hasAtLeastOneValue;
SingleValueNumericDocValuesFieldUpdates(long delGen, String field, int maxDoc, long value) {
- super(maxDoc, delGen, field, DocValuesType.NUMERIC);
+ this(delGen, field, DocValuesType.NUMERIC, maxDoc, value);
+ }
+
+ SingleValueNumericDocValuesFieldUpdates(
+ long delGen, String field, DocValuesType type, int maxDoc, long value) {
+ super(maxDoc, delGen, field, type);
this.bitSet = new SparseFixedBitSet(maxDoc);
this.value = value;
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/OverlaySortedNumericDocValues.java b/lucene/core/src/java/org/apache/lucene/index/OverlaySortedNumericDocValues.java
new file mode 100644
index 00000000000..d7d8a9bebb4
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/OverlaySortedNumericDocValues.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import java.io.IOException;
+import java.util.List;
+import org.apache.lucene.codecs.DocValuesProducer;
+
+/**
+ * Merges several {@link SortedNumericDocValues} layers (newest generation first, base last) into
+ * one view: for a given document the whole value set comes from the newest layer that has it, else
+ * the base. Unlike {@link OverlayNumericDocValues} this preserves multi-valued documents, so it
+ * works whether the base column is single- or multi-valued (a set-only sorted-numeric update writes
+ * a single-valued delta on top of a possibly multi-valued base).
+ */
+final class OverlaySortedNumericDocValues extends SortedNumericDocValues {
+
+ private final SortedNumericDocValues[] layers; // newest first, base last
+ private final DocValuesOverlayMerger merger;
+
+ OverlaySortedNumericDocValues(SortedNumericDocValues[] layers) {
+ assert layers != null && layers.length > 0;
+ this.layers = layers;
+ this.merger = new DocValuesOverlayMerger(layers);
+ }
+
+ @Override
+ public long nextValue() throws IOException {
+ return layers[merger.valueLayer()].nextValue();
+ }
+
+ @Override
+ public int docValueCount() {
+ return layers[merger.valueLayer()].docValueCount();
+ }
+
+ @Override
+ public boolean advanceExact(int target) throws IOException {
+ return merger.advanceExact(target);
+ }
+
+ @Override
+ public int docID() {
+ return merger.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ return merger.nextDoc();
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ return merger.advance(target);
+ }
+
+ @Override
+ public long cost() {
+ return merger.cost();
+ }
+
+ static SortedNumericDocValues from(FieldInfo fieldInfo, List<DocValuesProducer> producers)
+ throws IOException {
+ SortedNumericDocValues[] layers = new SortedNumericDocValues[producers.size()];
+ boolean allSingleValued = true;
+ for (int i = 0; i < layers.length; i++) {
+ layers[i] = producers.get(i).getSortedNumeric(fieldInfo);
+ allSingleValued &= DocValues.isSingleton(layers[i]);
+ }
+ if (allSingleValued) {
+ NumericDocValues[] singletonLayers = new NumericDocValues[producers.size()];
+ for (int i = 0; i < layers.length; i++) {
+ singletonLayers[i] = DocValues.unwrapSingleton(layers[i]);
+ }
+ return DocValues.singleton(new OverlayNumericDocValues(singletonLayers));
+ } else {
+ return new OverlaySortedNumericDocValues(layers);
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
index 80c062bbd42..4117d3bbec8 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
@@ -305,7 +305,9 @@ final class ReadersAndUpdates {
final String field = ent.getKey();
final List<DocValuesFieldUpdates> updates = ent.getValue();
DocValuesType type = updates.get(0).type;
- assert type == DocValuesType.NUMERIC || type == DocValuesType.BINARY
+ assert type == DocValuesType.NUMERIC
+ || type == DocValuesType.BINARY
+ || type == DocValuesType.SORTED_NUMERIC
: "unsupported type: " + type;
final List<DocValuesFieldUpdates> updatesToApply = new ArrayList<>();
long bytes = 0;
@@ -354,9 +356,10 @@ final class ReadersAndUpdates {
}
}
// Set-only updates become a sparse delta generation overlaid at read time; a removal falls
- // back to the dense
- // rewrite. (Skip-indexed fields can't reach here: IndexWriter rejects doc-values updates on
- // them.)
+ // back to the dense rewrite. (Skip-indexed fields can't reach here: IndexWriter rejects
+ // doc-values updates on them.) A single-valued sorted-numeric update writes a single-valued
+ // delta; the base column may be single- or multi-valued and is preserved per-doc by the
+ // sorted-numeric merge/overlay.
final boolean sparseDelta = config.getMaxDocValuesOverlays() > 0 && anyRemoval == false;
// Past maxDocValuesOverlays, fold the prior deltas into this write as one sparse generation
// (base untouched) instead of appending another.
@@ -384,9 +387,12 @@ final class ReadersAndUpdates {
// maxDoc for sparse columns) and deltas are small by construction.
deltaCoverage +=
docsWithValueCount(
- type == DocValuesType.BINARY
- ? p.getBinary(fieldInfo)
- : p.getNumeric(fieldInfo));
+ switch (type) {
+ case BINARY -> p.getBinary(fieldInfo);
+ case SORTED_NUMERIC -> p.getSortedNumeric(fieldInfo);
+ // $CASES-OMITTED$
+ default -> p.getNumeric(fieldInfo);
+ });
}
} catch (Throwable t) {
// The try-with-resources below owns the normal close; close here if opening a delta fails
@@ -483,7 +489,7 @@ final class ReadersAndUpdates {
};
}
});
- } else {
+ } else if (type == DocValuesType.NUMERIC) {
// write the numeric updates to a new gen'd docvalues file
fieldsConsumer.addNumericField(
fieldInfo,
@@ -501,49 +507,87 @@ final class ReadersAndUpdates {
compact && foldToDense == false
? overlayNumeric(deltaProducers, fieldInfo)
: reader.getNumericDocValues(field);
- final MergedDocValues<NumericDocValues> mergedDocValues =
- new MergedDocValues<>(
- onDisk,
- DocValuesFieldUpdates.Iterator.asNumericDocValues(iterator),
- iterator);
- // Merge sort of the original doc values with updated doc values:
- return new NumericDocValues() {
- @Override
- public long longValue() throws IOException {
- return mergedDocValues.currentValuesSupplier.longValue();
- }
-
- @Override
- public boolean advanceExact(int target) {
- return mergedDocValues.advanceExact(target);
- }
-
- @Override
- public int docID() {
- return mergedDocValues.docID();
- }
-
- @Override
- public int nextDoc() throws IOException {
- return mergedDocValues.nextDoc();
- }
-
- @Override
- public int advance(int target) {
- return mergedDocValues.advance(target);
- }
-
- @Override
- public void intoBitSet(int upTo, FixedBitSet bitSet, int offset)
- throws IOException {
- mergedDocValues.intoBitSet(upTo, bitSet, offset);
- }
-
- @Override
- public long cost() {
- return mergedDocValues.cost();
- }
- };
+ return handleMergedNumericDocValues(onDisk, iterator);
+ }
+ });
+ } else {
+ assert type == DocValuesType.SORTED_NUMERIC : "unsupported type: " + type;
+ // The update is single-valued but the base column may be multi-valued, so merge/overlay
+ // natively as sorted-numeric (the update wins per doc, other docs keep their whole set).
+ fieldsConsumer.addSortedNumericField(
+ fieldInfo,
+ new EmptyDocValuesProducer() {
+ @Override
+ public SortedNumericDocValues getSortedNumeric(FieldInfo fieldInfoIn)
+ throws IOException {
+ DocValuesFieldUpdates.Iterator iterator = updateSupplier.apply(fieldInfo);
+ if (sparseDelta && compact == false) {
+ // write only the updated docs (single-valued) as a singleton column; unchanged
+ // docs come from the base at read time
+ return DocValues.singleton(
+ DocValuesFieldUpdates.Iterator.asNumericDocValues(iterator));
+ }
+ // sparse fold merges the prior deltas (stays sparse); a dense rewrite (removal or
+ // fold-to-dense) merges the base column
+ final SortedNumericDocValues onDisk =
+ compact && foldToDense == false
+ ? overlaySortedNumeric(deltaProducers, fieldInfo)
+ : reader.getSortedNumericDocValues(field);
+ final NumericDocValues singletonOnDisk = DocValues.unwrapSingleton(onDisk);
+ if (singletonOnDisk != null) {
+ return DocValues.singleton(
+ handleMergedNumericDocValues(singletonOnDisk, iterator));
+ } else {
+ final MergedDocValues<SortedNumericDocValues> mergedDocValues =
+ new MergedDocValues<>(
+ onDisk,
+ DocValues.singleton(
+ DocValuesFieldUpdates.Iterator.asNumericDocValues(iterator)),
+ iterator);
+ // Merge sort of the original doc values with updated doc values:
+ return new SortedNumericDocValues() {
+ @Override
+ public long nextValue() throws IOException {
+ return mergedDocValues.currentValuesSupplier.nextValue();
+ }
+
+ @Override
+ public int docValueCount() {
+ return mergedDocValues.currentValuesSupplier.docValueCount();
+ }
+
+ @Override
+ public boolean advanceExact(int target) {
+ return mergedDocValues.advanceExact(target);
+ }
+
+ @Override
+ public int docID() {
+ return mergedDocValues.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ return mergedDocValues.nextDoc();
+ }
+
+ @Override
+ public int advance(int target) {
+ return mergedDocValues.advance(target);
+ }
+
+ @Override
+ public void intoBitSet(int upTo, FixedBitSet bitSet, int offset)
+ throws IOException {
+ mergedDocValues.intoBitSet(upTo, bitSet, offset);
+ }
+
+ @Override
+ public long cost() {
+ return mergedDocValues.cost();
+ }
+ };
+ }
}
});
}
@@ -605,6 +649,15 @@ final class ReadersAndUpdates {
return new OverlayBinaryDocValues(layers);
}
+ /**
+ * Fresh overlay over the delta generations of a sorted-numeric field, newest first, used to fold
+ * deltas during compaction.
+ */
+ private static SortedNumericDocValues overlaySortedNumeric(
+ List<DocValuesProducer> producers, FieldInfo fieldInfo) throws IOException {
+ return OverlaySortedNumericDocValues.from(fieldInfo, producers);
+ }
+
/**
* This class merges the current on-disk DV with an incoming update DV instance and merges the two
* instances giving the incoming update precedence in terms of values, in other words the values
@@ -741,6 +794,52 @@ final class ReadersAndUpdates {
}
}
+ private NumericDocValues handleMergedNumericDocValues(
+ NumericDocValues onDisk, DocValuesFieldUpdates.Iterator updateIterator) {
+ final MergedDocValues<NumericDocValues> mergedDocValues =
+ new MergedDocValues<>(
+ onDisk,
+ DocValuesFieldUpdates.Iterator.asNumericDocValues(updateIterator),
+ updateIterator);
+ // Merge sort of the original doc values with updated doc values:
+ return new NumericDocValues() {
+ @Override
+ public long longValue() throws IOException {
+ return mergedDocValues.currentValuesSupplier.longValue();
+ }
+
+ @Override
+ public boolean advanceExact(int target) {
+ return mergedDocValues.advanceExact(target);
+ }
+
+ @Override
+ public int docID() {
+ return mergedDocValues.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ return mergedDocValues.nextDoc();
+ }
+
+ @Override
+ public int advance(int target) {
+ return mergedDocValues.advance(target);
+ }
+
+ @Override
+ public void intoBitSet(int upTo, FixedBitSet bitSet, int offset) throws IOException {
+ mergedDocValues.intoBitSet(upTo, bitSet, offset);
+ }
+
+ @Override
+ public long cost() {
+ return mergedDocValues.cost();
+ }
+ };
+ }
+
private synchronized Set<String> writeFieldInfosGen(
FieldInfos fieldInfos, Directory dir, FieldInfosFormat infosFormat) throws IOException {
final long nextFieldInfosGen = info.getNextFieldInfosGen();
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java
index b3979856745..c20171c17e7 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java
@@ -17,6 +17,7 @@
package org.apache.lucene.index;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.IdentityHashMap;
@@ -199,7 +200,7 @@ class SegmentDocValuesProducer extends DocValuesProducer {
return new OverlayBinaryDocValues(layers);
}
- // sorted/sorted-set/sorted-numeric and skippers are never overlaid: only numeric/binary values
+ // sorted/sorted-set and skippers are never overlaid: only numeric/binary/sorted-numeric values
// can be updated in place, so these always have a single producer.
private DocValuesProducer single(FieldInfo field) {
DocValuesProducer[] producers = dvProducersByField.get(field.number);
@@ -215,7 +216,12 @@ class SegmentDocValuesProducer extends DocValuesProducer {
@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
- return single(field).getSortedNumeric(field);
+ DocValuesProducer[] producers = dvProducersByField.get(field.number);
+ assert producers != null;
+ if (producers.length == 1) {
+ return producers[0].getSortedNumeric(field);
+ }
+ return OverlaySortedNumericDocValues.from(field, Arrays.asList(producers));
}
@Override
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
index a1a1396a3dd..4c41d4e62b1 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
@@ -2017,6 +2017,13 @@ public class TestIndexSorting extends LuceneTestCase {
assertEquals(
exc.getMessage(),
"cannot update docvalues field involved in the index sort, field=foo, sort=<long: \"foo\">");
+ exc =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> w.updateSortedNumericDocValue(new Term("id", "0"), "foo", -1));
+ assertEquals(
+ exc.getMessage(),
+ "cannot update docvalues field involved in the index sort, field=foo, sort=<long: \"foo\">");
w.close();
dir.close();
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMergeCarryOverFromDisk.java b/lucene/core/src/test/org/apache/lucene/index/TestMergeCarryOverFromDisk.java
index 0cf456aa53e..46993a71891 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestMergeCarryOverFromDisk.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestMergeCarryOverFromDisk.java
@@ -17,12 +17,19 @@
package org.apache.lucene.index;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.StringField;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FilterDirectory;
import org.apache.lucene.store.IOContext;
@@ -67,10 +74,19 @@ public class TestMergeCarryOverFromDisk extends LuceneTestCase {
}
private static Document doc(int id, long ndv, String bdv) {
+ return doc(id, ndv, bdv, ndv);
+ }
+
+ /** As {@link #doc(int, long, String)} but with an explicit, possibly multi-valued, snv set. */
+ private static Document doc(int id, long ndv, String bdv, long... snv) {
Document d = new Document();
d.add(new StringField("id", Integer.toString(id), StringField.Store.NO));
d.add(new NumericDocValuesField("ndv", ndv));
d.add(new BinaryDocValuesField("bdv", new BytesRef(bdv)));
+ d.add(new SortedNumericDocValuesField("svsnv", ndv));
+ for (long v : snv) {
+ d.add(new SortedNumericDocValuesField("snv", v));
+ }
return d;
}
@@ -81,13 +97,17 @@ public class TestMergeCarryOverFromDisk extends LuceneTestCase {
.setMergeScheduler(new ConcurrentMergeScheduler());
IndexWriter writer = new IndexWriter(dir, conf);
- // Two segments; the default TieredMergePolicy won't auto-merge two small segments.
+ // Two segments; the default TieredMergePolicy won't auto-merge two small segments. docs 0-3 are
+ // single-valued and should create a NumericDocValues on disk; docs 4-7 are genuinely
+ // multi-valued ({i, i+1000}) and should create a SortedNumericDocValues on disk. A doc never
+ // updated for snv must keep its whole value set through the merge, while a doc updated for snv
+ // (3 and 7 below) collapses to the single update value.
for (int i = 0; i < 4; i++) {
- writer.addDocument(doc(i, i, "v" + i));
+ writer.addDocument(doc(i, i, "v" + i, i));
}
writer.commit();
for (int i = 4; i < 8; i++) {
- writer.addDocument(doc(i, i, "v" + i));
+ writer.addDocument(doc(i, i, "v" + i, i, i + 1000));
}
writer.commit();
@@ -109,12 +129,16 @@ public class TestMergeCarryOverFromDisk extends LuceneTestCase {
// output.
dir.mergeStarted.await();
- // Update numeric + binary values on docs in the now-merging segments, then resolve them to the
- // segments (and disk) by reopening a reader from the writer.
+ // Update numeric, binary, and sorted-numeric values on docs in the now-merging segments, then
+ // resolve them to the segments (and disk) by reopening a reader from the writer.
writer.updateNumericDocValue(new Term("id", "1"), "ndv", 101L);
writer.updateNumericDocValue(new Term("id", "5"), "ndv", 105L);
writer.updateBinaryDocValue(new Term("id", "2"), "bdv", new BytesRef("updated-2"));
writer.updateBinaryDocValue(new Term("id", "6"), "bdv", new BytesRef("updated-6"));
+ writer.updateSortedNumericDocValue(new Term("id", "3"), "snv", 103L);
+ writer.updateSortedNumericDocValue(new Term("id", "7"), "snv", 107L);
+ writer.updateSortedNumericDocValue(new Term("id", "3"), "svsnv", 103L);
+ writer.updateSortedNumericDocValue(new Term("id", "7"), "svsnv", 107L);
// Force resolution onto the merging segments (writes DV gens to disk).
try (DirectoryReader r = DirectoryReader.open(writer)) {
assertNotNull(r);
@@ -125,25 +149,76 @@ public class TestMergeCarryOverFromDisk extends LuceneTestCase {
merger.join();
assertNull("forceMerge failed: " + dir.failure.get(), dir.failure.get());
- // Verify the merged segment reflects the numeric + binary updates that resolved during the
- // merge.
+ // Verify every doc in the merged segment: the numeric, binary, and sorted-numeric updates that
+ // resolved during the merge landed on exactly their docs, and every untouched doc (including
+ // the
+ // multi-valued one) kept the value it had before the merge.
try (DirectoryReader reader = DirectoryReader.open(writer)) {
assertEquals(1, reader.leaves().size()); // single merged segment
LeafReader leaf = reader.leaves().get(0).reader();
NumericDocValues ndv = leaf.getNumericDocValues("ndv");
BinaryDocValues bdv = leaf.getBinaryDocValues("bdv");
- java.util.Set<Long> ndvValues = new java.util.HashSet<>();
- java.util.Set<String> bdvValues = new java.util.HashSet<>();
+ SortedNumericDocValues svsnv = leaf.getSortedNumericDocValues("svsnv");
+ assertTrue(
+ "single-valued SortedNumericDocValues should be a singleton",
+ DocValues.isSingleton(svsnv));
+ SortedNumericDocValues snv = leaf.getSortedNumericDocValues("snv");
+ assertFalse(
+ "multi-valued SortedNumericDocValues should not be a singleton",
+ DocValues.isSingleton(snv));
+ // docId -> id, so per-doc sorted-numeric value sets can be checked by id (postings iterate
+ // per term, not in doc order).
+ Map<Integer, String> idByDoc = new HashMap<>();
+ TermsEnum te = leaf.terms("id").iterator();
+ BytesRef t;
+ while ((t = te.next()) != null) {
+ PostingsEnum pe = te.postings(null, PostingsEnum.NONE);
+ for (int d = pe.nextDoc(); d != DocIdSetIterator.NO_MORE_DOCS; d = pe.nextDoc()) {
+ idByDoc.put(d, t.utf8ToString());
+ }
+ }
+ Map<String, Long> ndvById = new HashMap<>();
+ Map<String, String> bdvById = new HashMap<>();
+ Map<String, long[]> snvById = new HashMap<>();
for (int d = 0; d < leaf.maxDoc(); d++) {
+ String id = idByDoc.get(d);
assertTrue(ndv.advanceExact(d));
- ndvValues.add(ndv.longValue());
+ ndvById.put(id, ndv.longValue());
assertTrue(bdv.advanceExact(d));
- bdvValues.add(bdv.binaryValue().utf8ToString());
+ bdvById.put(id, bdv.binaryValue().utf8ToString());
+ assertTrue(snv.advanceExact(d)); // doc order, so advanceExact is monotonic
+ long[] vals = new long[snv.docValueCount()];
+ for (int i = 0; i < vals.length; i++) {
+ vals[i] = snv.nextValue();
+ }
+ snvById.put(id, vals);
+ }
+ // Every doc must hold exactly its expected value after the merge: the four updated docs carry
+ // the new value, and every untouched doc keeps what it had before the merge.
+ for (int id = 0; id < 8; id++) {
+ String key = Integer.toString(id);
+ long expectedNdv =
+ switch (id) {
+ case 1 -> 101L;
+ case 5 -> 105L;
+ default -> id;
+ };
+ assertEquals("ndv for doc " + id, expectedNdv, (long) ndvById.get(key));
+ String expectedBdv =
+ switch (id) {
+ case 2 -> "updated-2";
+ case 6 -> "updated-6";
+ default -> "v" + id;
+ };
+ assertEquals("bdv for doc " + id, expectedBdv, bdvById.get(key));
+ long[] expectedSnv =
+ switch (id) {
+ case 3 -> new long[] {103};
+ case 7 -> new long[] {107};
+ default -> id < 4 ? new long[] {id} : new long[] {id, id + 1000};
+ };
+ assertArrayEquals("snv for doc " + id, expectedSnv, snvById.get(key));
}
- assertTrue("carried numeric update 101 missing", ndvValues.contains(101L));
- assertTrue("carried numeric update 105 missing", ndvValues.contains(105L));
- assertTrue("carried binary update updated-2 missing", bdvValues.contains("updated-2"));
- assertTrue("carried binary update updated-6 missing", bdvValues.contains("updated-6"));
}
writer.close();
@@ -163,12 +238,14 @@ public class TestMergeCarryOverFromDisk extends LuceneTestCase {
// small overlay budget so a handful of updates already triggers a fold
.setMaxDocValuesOverlays(4);
IndexWriter writer = new IndexWriter(dir, conf);
+ // Segment 1 (docs 0-3): single-valued snv, stored as a numeric column. Segment 2 (docs 4-7):
+ // multi-valued snv ({i, i+1000}), stored as a sorted-numeric column.
for (int i = 0; i < 4; i++) {
- writer.addDocument(doc(i, i, "v" + i));
+ writer.addDocument(doc(i, i, "v" + i, i));
}
writer.commit();
for (int i = 4; i < 8; i++) {
- writer.addDocument(doc(i, i, "v" + i));
+ writer.addDocument(doc(i, i, "v" + i, i, i + 1000));
}
writer.commit();
@@ -186,12 +263,16 @@ public class TestMergeCarryOverFromDisk extends LuceneTestCase {
dir.mergeStarted.await();
// Many updates to doc 3 (in the first, merging segment), each resolved separately so it becomes
- // a
- // new overlay generation; well past maxDocValuesOverlays, so folds happen mid-merge.
+ // a new overlay generation; well past maxDocValuesOverlays, so folds happen mid-merge. Both a
+ // numeric and a sorted-numeric field are folded so the carry-over reads back a folded column of
+ // each kind.
long finalValue = -1;
+ long finalSnv = -1;
for (int v = 0; v < 20; v++) {
finalValue = 1000L + v;
+ finalSnv = 2000L + v;
writer.updateNumericDocValue(new Term("id", "3"), "ndv", finalValue);
+ writer.updateSortedNumericDocValue(new Term("id", "3"), "snv", finalSnv);
try (DirectoryReader r = DirectoryReader.open(writer)) {
assertNotNull(r);
}
@@ -203,13 +284,21 @@ public class TestMergeCarryOverFromDisk extends LuceneTestCase {
try (DirectoryReader reader = DirectoryReader.open(writer)) {
assertEquals(1, reader.leaves().size());
- NumericDocValues ndv = reader.leaves().get(0).reader().getNumericDocValues("ndv");
- java.util.Set<Long> values = new java.util.HashSet<>();
- for (int d = 0; d < reader.leaves().get(0).reader().maxDoc(); d++) {
+ LeafReader leaf = reader.leaves().get(0).reader();
+ NumericDocValues ndv = leaf.getNumericDocValues("ndv");
+ SortedNumericDocValues snv = leaf.getSortedNumericDocValues("snv");
+ Set<Long> values = new HashSet<>();
+ Set<Long> snvValues = new HashSet<>();
+ for (int d = 0; d < leaf.maxDoc(); d++) {
assertTrue(ndv.advanceExact(d));
values.add(ndv.longValue());
+ assertTrue(snv.advanceExact(d));
+ for (int i = 0, count = snv.docValueCount(); i < count; i++) {
+ snvValues.add(snv.nextValue());
+ }
}
assertTrue("folded final value " + finalValue + " missing", values.contains(finalValue));
+ assertTrue("folded final snv " + finalSnv + " missing", snvValues.contains(finalSnv));
}
writer.close();
@@ -229,12 +318,14 @@ public class TestMergeCarryOverFromDisk extends LuceneTestCase {
.setMergeScheduler(new ConcurrentMergeScheduler())
.setSoftDeletesField("__soft");
IndexWriter writer = new IndexWriter(dir, conf);
+ // Segment 1 (docs 0-3): single-valued snv, stored as a numeric column. Segment 2 (docs 4-7):
+ // multi-valued snv ({i, i+1000}), stored as a sorted-numeric column.
for (int i = 0; i < 4; i++) {
- writer.addDocument(doc(i, i, "v" + i));
+ writer.addDocument(doc(i, i, "v" + i, i));
}
writer.commit();
for (int i = 4; i < 8; i++) {
- writer.addDocument(doc(i, i, "v" + i));
+ writer.addDocument(doc(i, i, "v" + i, i, i + 1000));
}
writer.commit();
@@ -256,6 +347,10 @@ public class TestMergeCarryOverFromDisk extends LuceneTestCase {
Document replacement = doc(1, 101, "updated-1");
writer.softUpdateDocument(
new Term("id", "1"), replacement, new NumericDocValuesField("__soft", 1));
+ // Also update a sorted-numeric value on a surviving multi-valued doc (id 5, in segment 2), to
+ // confirm the update collapses it to a single value and is carried over alongside the
+ // soft-delete.
+ writer.updateSortedNumericDocValue(new Term("id", "5"), "snv", 205L);
try (DirectoryReader r = DirectoryReader.open(writer)) {
assertNotNull(r);
}
@@ -265,23 +360,39 @@ public class TestMergeCarryOverFromDisk extends LuceneTestCase {
assertNull("forceMerge failed: " + dir.failure.get(), dir.failure.get());
// Exactly one doc with id=1 must be live (the replacement); the old one must be soft-deleted.
+ // The replacement doc carries its own snv, and the sorted-numeric update on doc 5 is carried
+ // over too.
try (DirectoryReader reader = DirectoryReader.open(writer)) {
int live = 0;
+ boolean sawCarriedSnv = false;
for (LeafReaderContext ctx : reader.leaves()) {
LeafReader leaf = ctx.reader();
NumericDocValues ndv = leaf.getNumericDocValues("ndv");
+ SortedNumericDocValues snv = leaf.getSortedNumericDocValues("snv");
Bits liveDocs = leaf.getLiveDocs();
for (int d = 0; d < leaf.maxDoc(); d++) {
if (liveDocs != null && liveDocs.get(d) == false) {
continue;
}
assertTrue(ndv.advanceExact(d));
+ assertTrue(snv.advanceExact(d)); // doc order, so advanceExact is monotonic
+ long[] snvVals = new long[snv.docValueCount()];
+ for (int i = 0; i < snvVals.length; i++) {
+ snvVals[i] = snv.nextValue();
+ }
if (ndv.longValue() == 101) {
live++;
+ // the replacement doc is in the single-valued segment
+ assertArrayEquals("replacement doc snv", new long[] {101}, snvVals);
+ }
+ // doc 5 (multi-valued base) collapsed to a single value by the carried update
+ if (Arrays.equals(snvVals, new long[] {205})) {
+ sawCarriedSnv = true;
}
}
}
assertEquals("exactly one live replacement doc expected", 1, live);
+ assertTrue("carried sorted-numeric update 205 missing", sawCarriedSnv);
}
writer.close();
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSortedNumericDocValuesUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestSortedNumericDocValuesUpdates.java
new file mode 100644
index 00000000000..b98e1eb1d1a
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSortedNumericDocValuesUpdates.java
@@ -0,0 +1,669 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeMap;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedNumericSortField;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.analysis.MockAnalyzer;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.IOUtils;
+
+/**
+ * Tests in-place updates of SORTED_NUMERIC doc-values. An update sets a single value per matched
+ * document, replacing whatever value(s) that document had; the field itself may be single- or
+ * multi-valued, and documents not matched by the update keep their existing values. A single-valued
+ * column is stored as a numeric column and reuses the numeric update machinery; a multi-valued
+ * column is preserved and merged per-doc.
+ */
+public class TestSortedNumericDocValuesUpdates extends LuceneTestCase {
+
+ private Document doc(int id, long... values) {
+ Document doc = new Document();
+ doc.add(new StringField("id", "doc-" + id, Store.NO));
+ for (long v : values) {
+ doc.add(new SortedNumericDocValuesField("val", v));
+ }
+ return doc;
+ }
+
+ private IndexWriterConfig conf() {
+ IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
+ // exercise both the classic dense rewrite and the sparse-delta overlay
+ conf.setMaxDocValuesOverlays(random().nextBoolean() ? 0 : 1 + random().nextInt(4));
+ return conf;
+ }
+
+ public void testSimpleReplace() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, conf());
+ writer.addDocument(doc(0, 3));
+ writer.addDocument(doc(1, 5));
+ writer.commit();
+
+ writer.updateSortedNumericDocValue(new Term("id", "doc-0"), "val", 9);
+ try (DirectoryReader reader = DirectoryReader.open(writer)) {
+ Map<String, long[]> values = collect(reader);
+ assertArrayEquals(new long[] {9}, values.get("doc-0"));
+ assertArrayEquals(new long[] {5}, values.get("doc-1"));
+ }
+ IOUtils.close(writer, dir);
+ }
+
+ public void testUpdateViaFieldsApi() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, conf());
+ writer.addDocument(doc(0, 1));
+ writer.commit();
+
+ writer.updateDocValues(new Term("id", "doc-0"), new SortedNumericDocValuesField("val", 42));
+ try (DirectoryReader reader = DirectoryReader.open(writer)) {
+ assertArrayEquals(new long[] {42}, collect(reader).get("doc-0"));
+ }
+ IOUtils.close(writer, dir);
+ }
+
+ public void testMultipleFieldsSameNameLastWins() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, conf());
+ writer.addDocument(doc(0, 1));
+ writer.commit();
+ // two same-named single-valued sorted-numeric fields in one update behave like numeric updates:
+ // each is applied and the last one wins (this is not a multi-valued update).
+ writer.updateDocValues(
+ new Term("id", "doc-0"),
+ new SortedNumericDocValuesField("val", 1),
+ new SortedNumericDocValuesField("val", 2));
+ try (DirectoryReader reader = DirectoryReader.open(writer)) {
+ assertArrayEquals(new long[] {2}, collect(reader).get("doc-0"));
+ }
+ IOUtils.close(writer, dir);
+ }
+
+ public void testUpdateOverMultiValuedBase() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, conf());
+ writer.addDocument(doc(0, 1, 2, 3)); // genuinely multi-valued base
+ writer.addDocument(doc(1, 7, 8));
+ writer.commit();
+
+ // updating doc-0 to a single value succeeds even though the column is multi-valued; doc-1 keeps
+ // its whole value set.
+ writer.updateSortedNumericDocValue(new Term("id", "doc-0"), "val", 99);
+ try (DirectoryReader reader = DirectoryReader.open(writer)) {
+ Map<String, long[]> values = collect(reader);
+ assertArrayEquals(new long[] {99}, values.get("doc-0"));
+ assertArrayEquals(new long[] {7, 8}, values.get("doc-1"));
+ }
+ IOUtils.close(writer, dir);
+ }
+
+ public void testUpdateKeepsOtherDocsMultiValuedAcrossMerge() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = conf();
+ conf.setMaxBufferedDocs(2);
+ IndexWriter writer = new IndexWriter(dir, conf);
+ for (int i = 0; i < 6; i++) {
+ writer.addDocument(doc(i, i * 10L, i * 10L + 1, i * 10L + 2)); // 3 values each
+ if (i % 2 == 1) {
+ writer.commit();
+ }
+ }
+ // update a few docs to a single value; the rest must keep their 3-value sets, through a merge.
+ writer.updateSortedNumericDocValue(new Term("id", "doc-1"), "val", 500);
+ writer.updateSortedNumericDocValue(new Term("id", "doc-4"), "val", 501);
+ writer.forceMerge(1);
+ try (DirectoryReader reader = DirectoryReader.open(writer)) {
+ Map<String, long[]> values = collect(reader);
+ for (int i = 0; i < 6; i++) {
+ if (i == 1) {
+ assertArrayEquals(new long[] {500}, values.get("doc-1"));
+ } else if (i == 4) {
+ assertArrayEquals(new long[] {501}, values.get("doc-4"));
+ } else {
+ assertArrayEquals(new long[] {i * 10L, i * 10L + 1, i * 10L + 2}, values.get("doc-" + i));
+ }
+ }
+ }
+ IOUtils.close(writer, dir);
+ }
+
+ public void testDenseRewriteOverMultiValuedBase() throws Exception {
+ // maxDocValuesOverlays == 0 forces a dense full-column rewrite on every update. With a
+ // multi-valued base the rewrite merges the base column with the single-valued update per doc
+ // (the update winning): the matched doc collapses to the single update value while every other
+ // doc keeps its whole (sorted) value set read from the base column.
+ try (Directory dir = newDirectory();
+ IndexWriter w =
+ new IndexWriter(
+ dir,
+ new IndexWriterConfig(new MockAnalyzer(random())).setMaxDocValuesOverlays(0))) {
+ w.addDocument(doc(0, 5, 1, 3)); // multi-valued, unsorted input
+ w.addDocument(doc(1, 9, 8));
+ w.addDocument(doc(2, 2)); // single-valued
+ w.commit();
+
+ w.updateSortedNumericDocValue(new Term("id", "doc-1"), "val", 42);
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ Map<String, long[]> values = collect(reader);
+ assertArrayEquals(new long[] {1, 3, 5}, values.get("doc-0")); // untouched, sorted
+ assertArrayEquals(new long[] {42}, values.get("doc-1")); // replaced with a single value
+ assertArrayEquals(new long[] {2}, values.get("doc-2")); // untouched
+ }
+ // overlays are disabled, so the update produced a dense column, not a sparse delta.
+ SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
+ assertTrue(sis.info(0).getDocValuesOverlays().isEmpty());
+ TestUtil.checkIndex(dir);
+ }
+ }
+
+ public void testFoldToDenseOverMultiValuedBase() throws Exception {
+ // Drive a fold-to-dense rewrite over a genuinely multi-valued base. With
+ // maxDocValuesOverlays==1
+ // the first round writes a sparse delta; once its coverage crosses
+ // FOLD_TO_DENSE_COVERAGE_RATIO (0.5) the next update folds the field back to a single dense
+ // column, merging the single-valued update over the multi-valued base per doc. Docs never
+ // updated
+ // must keep their full value sets.
+ int numDocs = 10;
+ try (Directory dir = newDirectory();
+ IndexWriter w =
+ new IndexWriter(
+ dir,
+ new IndexWriterConfig(new MockAnalyzer(random()))
+ .setMaxDocValuesOverlays(1)
+ .setMergePolicy(NoMergePolicy.INSTANCE))) {
+ for (int i = 0; i < numDocs; i++) {
+ w.addDocument(doc(i, i, i + 100)); // two values each; sorted view is {i, i+100}
+ }
+ w.commit();
+
+ // Round 1: update 7 of 10 docs (> 50% coverage) so the delta will trigger a fold on the next
+ // write. Docs 7, 8, 9 are never touched and stay multi-valued in the base.
+ Map<String, long[]> expected = new HashMap<>();
+ for (int i = 0; i < numDocs; i++) {
+ expected.put("doc-" + i, new long[] {i, i + 100L});
+ }
+ for (int i = 0; i <= 6; i++) {
+ w.updateSortedNumericDocValue(new Term("id", "doc-" + i), "val", 1000L + i);
+ expected.put("doc-" + i, new long[] {1000L + i});
+ }
+ w.commit();
+
+ // Round 2: a single further update now folds to a dense column over the multi-valued base.
+ w.updateSortedNumericDocValue(new Term("id", "doc-0"), "val", 2000L);
+ expected.put("doc-0", new long[] {2000L});
+ w.commit();
+
+ // The fold collapses the field back to one dense column, clearing the sparse overlay.
+ SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
+ assertEquals(1, sis.size());
+ assertTrue(
+ "fold-to-dense should have cleared the overlay",
+ sis.info(0).getDocValuesOverlays().isEmpty());
+
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ Map<String, long[]> values = collect(reader);
+ for (Map.Entry<String, long[]> e : expected.entrySet()) {
+ assertArrayEquals(e.getKey(), e.getValue(), values.get(e.getKey()));
+ }
+ }
+ TestUtil.checkIndex(dir);
+ }
+ }
+
+ public void testUpdateThenDelete() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, conf());
+ writer.addDocument(doc(0, 1));
+ writer.addDocument(doc(1, 3));
+ writer.commit();
+
+ writer.updateSortedNumericDocValue(new Term("id", "doc-0"), "val", 100);
+ writer.deleteDocuments(new Term("id", "doc-1"));
+ try (DirectoryReader reader = DirectoryReader.open(writer)) {
+ Map<String, long[]> values = collect(reader);
+ assertArrayEquals(new long[] {100}, values.get("doc-0"));
+ assertFalse(values.containsKey("doc-1"));
+ }
+ IOUtils.close(writer, dir);
+ }
+
+ public void testNRTReopenSeesUpdates() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, conf());
+ writer.addDocument(doc(0, 1));
+ try (DirectoryReader r1 = DirectoryReader.open(writer)) {
+ assertArrayEquals(new long[] {1}, collect(r1).get("doc-0"));
+ writer.updateSortedNumericDocValue(new Term("id", "doc-0"), "val", 8);
+ try (DirectoryReader r2 = DirectoryReader.openIfChanged(r1)) {
+ assertNotNull(r2);
+ assertArrayEquals(new long[] {8}, collect(r2).get("doc-0"));
+ }
+ }
+ IOUtils.close(writer, dir);
+ }
+
+ public void testStackedUpdatesNewestWins() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = conf();
+ conf.setMaxDocValuesOverlays(4); // force the overlay path with several stacked deltas
+ IndexWriter writer = new IndexWriter(dir, conf);
+ writer.addDocument(doc(0, 1));
+ writer.addDocument(doc(1, 2));
+ writer.commit();
+ for (int gen = 0; gen < 5; gen++) {
+ writer.updateSortedNumericDocValue(new Term("id", "doc-0"), "val", gen * 10L);
+ writer.commit();
+ }
+ try (DirectoryReader reader = DirectoryReader.open(writer)) {
+ assertArrayEquals(new long[] {40}, collect(reader).get("doc-0"));
+ assertArrayEquals(new long[] {2}, collect(reader).get("doc-1"));
+ }
+ IOUtils.close(writer, dir);
+ }
+
+ public void testAcrossSegmentsAndMerge() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = conf();
+ conf.setMaxBufferedDocs(2);
+ IndexWriter writer = new IndexWriter(dir, conf);
+ for (int i = 0; i < 10; i++) {
+ writer.addDocument(doc(i, i));
+ if (i % 2 == 1) {
+ writer.commit();
+ }
+ }
+ for (int i = 0; i < 10; i++) {
+ writer.updateSortedNumericDocValue(new Term("id", "doc-" + i), "val", i * 100L);
+ }
+ writer.forceMerge(1);
+ try (DirectoryReader reader = DirectoryReader.open(writer)) {
+ Map<String, long[]> values = collect(reader);
+ for (int i = 0; i < 10; i++) {
+ assertArrayEquals("doc-" + i, new long[] {i * 100L}, values.get("doc-" + i));
+ }
+ }
+ IOUtils.close(writer, dir);
+ }
+
+ public void testRejectWrongType() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, conf());
+ writer.addDocument(doc(0, 1));
+ writer.commit();
+ // "val" is SORTED_NUMERIC; trying to update it as NUMERIC must fail
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> writer.updateNumericDocValue(new Term("id", "doc-0"), "val", 5L));
+ IOUtils.close(writer, dir);
+ }
+
+ public void testRejectIndexSortField() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = conf();
+ conf.setIndexSort(new Sort(new SortedNumericSortField("val", SortField.Type.LONG)));
+ IndexWriter writer = new IndexWriter(dir, conf);
+ writer.addDocument(doc(0, 1));
+ writer.commit();
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> writer.updateSortedNumericDocValue(new Term("id", "doc-0"), "val", 5L));
+ IOUtils.close(writer, dir);
+ }
+
+ public void testFoldsToDenseOnFullCorpusUpdates() throws Exception {
+ // Mirrors TestIncrementalDocValuesUpdates#testFoldsToDenseOnFullCorpusUpdates for the
+ // single-valued sorted-numeric column, which reuses the numeric overlay/fold machinery: once
+ // the
+ // stacked deltas cover the whole corpus the field folds back to a single dense column.
+ try (Directory dir = newDirectory();
+ IndexWriter w =
+ new IndexWriter(
+ dir,
+ new IndexWriterConfig(new MockAnalyzer(random())).setMaxDocValuesOverlays(2))) {
+ int numDocs = 40;
+ for (int i = 0; i < numDocs; i++) {
+ w.addDocument(doc(i, i));
+ }
+ long expected = 0;
+ for (int round = 0; round < 12; round++) {
+ expected = 1000L + round;
+ for (int i = 0; i < numDocs; i++) {
+ w.updateSortedNumericDocValue(new Term("id", "doc-" + i), "val", expected + i);
+ }
+ w.commit();
+ }
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ Map<String, long[]> values = collect(reader);
+ for (int i = 0; i < numDocs; i++) {
+ assertArrayEquals(new long[] {expected + i}, values.get("doc-" + i));
+ }
+ }
+ TestUtil.checkIndex(dir);
+ }
+ }
+
+ public void testSparseFoldOverDenseBase() throws Exception {
+ // Mirrors TestIncrementalDocValuesUpdates#testSparseFoldOverDenseBase: after a fold-to-dense
+ // the
+ // field's base becomes a dense generation (!= -1), and further updates must keep stacking a
+ // sparse overlay over that dense base rather than clearing it. The singleton wrap/unwrap of the
+ // sorted-numeric fold must preserve this.
+ int numDocs = 10;
+ try (Directory dir = newDirectory();
+ IndexWriter w =
+ new IndexWriter(
+ dir,
+ new IndexWriterConfig(new MockAnalyzer(random()))
+ .setMaxDocValuesOverlays(1)
+ .setMergePolicy(NoMergePolicy.INSTANCE))) {
+ for (int i = 0; i < numDocs; i++) {
+ w.addDocument(doc(i, i));
+ }
+ w.commit();
+ // Update every doc twice: the second round's coverage reaches maxDoc and folds to a dense
+ // column, so the field's base becomes a dense generation rather than the core column.
+ for (int round = 1; round <= 2; round++) {
+ for (int i = 0; i < numDocs; i++) {
+ w.updateSortedNumericDocValue(new Term("id", "doc-" + i), "val", 100L * round + i);
+ }
+ w.commit();
+ }
+ // Now keep updating a single doc across two more folds over the dense base.
+ for (int round = 0; round < 2; round++) {
+ w.updateSortedNumericDocValue(new Term("id", "doc-0"), "val", 999L + round);
+ w.commit();
+ }
+ // The last fold over the dense base must stay sparse: an overlay whose base is a dense
+ // generation (!= -1) with a folded delta, not a full-column rewrite that clears the overlay.
+ SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
+ assertEquals(1, sis.size());
+ Map<Integer, long[]> overlays = sis.info(0).getDocValuesOverlays();
+ assertFalse("expected a sparse overlay over the dense base", overlays.isEmpty());
+ long[] packed = overlays.values().iterator().next();
+ assertTrue("base should be a dense generation, not the core column", packed[0] != -1);
+ assertTrue("expected at least one delta over the dense base", packed.length >= 2);
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ Map<String, long[]> values = collect(reader);
+ assertArrayEquals(new long[] {1000L}, values.get("doc-0"));
+ for (int i = 1; i < numDocs; i++) {
+ assertArrayEquals(new long[] {200L + i}, values.get("doc-" + i));
+ }
+ }
+ TestUtil.checkIndex(dir);
+ }
+ }
+
+ public void testRandom() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, conf());
+ int numDocs = atLeast(50);
+ Map<String, long[]> expected = new HashMap<>();
+ for (int i = 0; i < numDocs; i++) {
+ // start single-valued so every doc can be updated in place
+ long v = random().nextInt(1000);
+ writer.addDocument(doc(i, v));
+ expected.put("doc-" + i, new long[] {v});
+ if (rarely()) {
+ writer.commit();
+ }
+ }
+ int numUpdates = atLeast(100);
+ for (int u = 0; u < numUpdates; u++) {
+ int id = random().nextInt(numDocs);
+ long v = random().nextInt(1000);
+ writer.updateSortedNumericDocValue(new Term("id", "doc-" + id), "val", v);
+ expected.put("doc-" + id, new long[] {v});
+ if (rarely()) {
+ writer.commit();
+ }
+ if (rarely()) {
+ writer.forceMerge(1 + random().nextInt(3));
+ }
+ }
+ try (DirectoryReader reader = DirectoryReader.open(writer)) {
+ Map<String, long[]> actual = collect(reader);
+ for (Map.Entry<String, long[]> e : expected.entrySet()) {
+ assertArrayEquals(e.getKey(), e.getValue(), actual.get(e.getKey()));
+ }
+ }
+ IOUtils.close(writer, dir);
+ }
+
+ public void testRandomMultiValued() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
+ // Low overlay budget (0..2) so stacked deltas frequently fold to a dense rewrite over the
+ // multi-valued base, exercising the base+update dense merge against the model.
+ conf.setMaxDocValuesOverlays(random().nextInt(3));
+ IndexWriter writer = new IndexWriter(dir, conf);
+ int numDocs = atLeast(50);
+ Map<String, long[]> expected = new HashMap<>();
+ for (int i = 0; i < numDocs; i++) {
+ // Seed genuinely multi-valued docs (1..4 values). Many docs are never updated below, so they
+ // keep their multi-valued sets and force the dense rewrites to read a multi-valued base.
+ long[] vals = new long[1 + random().nextInt(4)];
+ for (int j = 0; j < vals.length; j++) {
+ vals[j] = random().nextInt(1000);
+ }
+ writer.addDocument(doc(i, vals));
+ long[] sorted = vals.clone();
+ Arrays.sort(sorted); // a sorted-numeric column returns its values ascending, keeping dups
+ expected.put("doc-" + i, sorted);
+ if (rarely()) {
+ writer.commit();
+ }
+ }
+ int numUpdates = atLeast(100);
+ for (int u = 0; u < numUpdates; u++) {
+ // Only update a subset of docs so a good fraction stay multi-valued in the base.
+ int id = random().nextInt(numDocs);
+ long v = random().nextInt(1000);
+ writer.updateSortedNumericDocValue(new Term("id", "doc-" + id), "val", v);
+ expected.put("doc-" + id, new long[] {v}); // an update replaces the whole set with one value
+ if (rarely()) {
+ writer.commit();
+ }
+ if (rarely()) {
+ writer.forceMerge(1 + random().nextInt(3));
+ }
+ }
+ try (DirectoryReader reader = DirectoryReader.open(writer)) {
+ Map<String, long[]> actual = collect(reader);
+ for (Map.Entry<String, long[]> e : expected.entrySet()) {
+ assertArrayEquals(e.getKey(), e.getValue(), actual.get(e.getKey()));
+ }
+ }
+ IOUtils.close(writer, dir);
+ }
+
+ public void testMixedSingleAndMultiValuedSegments() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, conf());
+
+ // Alternate segments: even segments hold only single-valued "val" docs (a singleton column on
+ // disk), odd segments hold multi-valued "val" docs (a true sorted-numeric column). Every doc
+ // also has an always-single-valued "sv" field. Merging then combines singleton and
+ // non-singleton columns for "val", while "sv" stays single-valued everywhere.
+ int numSegments = 6;
+ int docsPerSegment = 5;
+ Map<String, long[]> expectedVal = new HashMap<>();
+ Map<String, long[]> expectedSv = new HashMap<>();
+ int id = 0;
+ for (int s = 0; s < numSegments; s++) {
+ boolean multiValued = (s % 2 == 1);
+ for (int d = 0; d < docsPerSegment; d++, id++) {
+ Document doc = new Document();
+ doc.add(new StringField("id", "doc-" + id, Store.NO));
+ long sv = id * 7L;
+ doc.add(new SortedNumericDocValuesField("sv", sv));
+ expectedSv.put("doc-" + id, new long[] {sv});
+ long[] vals = multiValued ? new long[] {id, id + 1000L} : new long[] {id};
+ for (long v : vals) {
+ doc.add(new SortedNumericDocValuesField("val", v));
+ }
+ long[] sorted = vals.clone();
+ Arrays.sort(sorted);
+ expectedVal.put("doc-" + id, sorted);
+ writer.addDocument(doc);
+ }
+ writer.commit(); // one segment per iteration
+ }
+ int totalDocs = id;
+
+ // Update every third doc's "val" (a single-valued update: collapses a multi-valued doc, or just
+ // changes a single-valued one) and its "sv" (which stays single-valued).
+ for (int u = 0; u < totalDocs; u += 3) {
+ writer.updateSortedNumericDocValue(new Term("id", "doc-" + u), "val", 5000L + u);
+ expectedVal.put("doc-" + u, new long[] {5000L + u});
+ writer.updateSortedNumericDocValue(new Term("id", "doc-" + u), "sv", 9000L + u);
+ expectedSv.put("doc-" + u, new long[] {9000L + u});
+ }
+
+ writer.forceMerge(1);
+
+ try (DirectoryReader reader = DirectoryReader.open(writer)) {
+ assertEquals(1, reader.leaves().size());
+ LeafReader leaf = reader.leaves().get(0).reader();
+ // "val" mixes multi-valued docs from the odd segments, so its merged column is not a
+ // singleton.
+ assertFalse(
+ "val column should be multi-valued",
+ DocValues.isSingleton(leaf.getSortedNumericDocValues("val")));
+ // "sv" was single-valued on every doc and only ever updated with a single value, so its
+ // merged column must be a singleton.
+ assertTrue(
+ "sv column should be a singleton",
+ DocValues.isSingleton(leaf.getSortedNumericDocValues("sv")));
+
+ Map<String, long[]> actualVal = collect(reader, "val");
+ Map<String, long[]> actualSv = collect(reader, "sv");
+ for (int i = 0; i < totalDocs; i++) {
+ String key = "doc-" + i;
+ assertArrayEquals(key + " val", expectedVal.get(key), actualVal.get(key));
+ assertArrayEquals(key + " sv", expectedSv.get(key), actualSv.get(key));
+ }
+ }
+ writer.commit(); // persist the merged segment + updates so checkIndex validates them
+ TestUtil.checkIndex(dir);
+ IOUtils.close(writer, dir);
+ }
+
+ public void testUpdateAllToSingleValueBecomesSingletonAfterMerge() throws Exception {
+ Directory dir = newDirectory();
+ // Overlays on, so the single-valued updates are written as sparse deltas over the multi-valued
+ // base rather than as a dense rewrite.
+ IndexWriterConfig conf =
+ newIndexWriterConfig(new MockAnalyzer(random()))
+ .setMaxDocValuesOverlays(1 + random().nextInt(4));
+ IndexWriter writer = new IndexWriter(dir, conf);
+ int numDocs = 20;
+ for (int i = 0; i < numDocs; i++) {
+ writer.addDocument(doc(i, i, i + 100L, i + 200L)); // 3 values each -> multi-valued column
+ }
+ writer.commit();
+
+ // Update every doc to a single value.
+ Map<String, long[]> expected = new HashMap<>();
+ for (int i = 0; i < numDocs; i++) {
+ writer.updateSortedNumericDocValue(new Term("id", "doc-" + i), "val", 1000L + i);
+ expected.put("doc-" + i, new long[] {1000L + i});
+ }
+
+ // Add one more (single-valued) doc in its own segment: this guarantees at least two segments,
+ // so forceMerge(1) actually runs a merge (a forceMerge of a single segment is a no-op and would
+ // leave the sparse overlay in place, which reports non-singleton because its base layer is
+ // multi-valued). The merge materializes everything into one dense column; every doc now holds a
+ // single value, so the codec stores it without an addresses table and it reads back as a
+ // singleton.
+ writer.addDocument(doc(numDocs, 5000L));
+ expected.put("doc-" + numDocs, new long[] {5000L});
+ writer.commit();
+ writer.forceMerge(1);
+
+ try (DirectoryReader reader = DirectoryReader.open(writer)) {
+ assertEquals(1, reader.leaves().size());
+ LeafReader leaf = reader.leaves().get(0).reader();
+ assertTrue(
+ "a merged column whose every doc holds a single value should be a singleton",
+ DocValues.isSingleton(leaf.getSortedNumericDocValues("val")));
+ Map<String, long[]> actual = collect(reader);
+ for (int i = 0; i <= numDocs; i++) {
+ assertArrayEquals("doc-" + i, expected.get("doc-" + i), actual.get("doc-" + i));
+ }
+ }
+ writer.commit();
+ TestUtil.checkIndex(dir);
+ IOUtils.close(writer, dir);
+ }
+
+ /** Collect field "val" for all live docs, keyed by the "id" stored term. */
+ private static Map<String, long[]> collect(DirectoryReader reader) throws Exception {
+ return collect(reader, "val");
+ }
+
+ /** Collect the given sorted-numeric field for all live docs, keyed by the "id" stored term. */
+ private static Map<String, long[]> collect(DirectoryReader reader, String field)
+ throws Exception {
+ Map<String, long[]> result = new TreeMap<>();
+ for (LeafReaderContext ctx : reader.leaves()) {
+ LeafReader lr = ctx.reader();
+ SortedNumericDocValues dv = lr.getSortedNumericDocValues(field);
+ Terms idTerms = lr.terms("id");
+ if (idTerms == null || dv == null) {
+ continue;
+ }
+ Map<Integer, String> idByDoc = new HashMap<>();
+ TermsEnum te = idTerms.iterator();
+ org.apache.lucene.util.BytesRef term;
+ while ((term = te.next()) != null) {
+ PostingsEnum pe = te.postings(null, PostingsEnum.NONE);
+ int d;
+ while ((d = pe.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
+ idByDoc.put(d, term.utf8ToString());
+ }
+ }
+ Bits liveDocs = lr.getLiveDocs();
+ int doc;
+ while ((doc = dv.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
+ if (liveDocs != null && liveDocs.get(doc) == false) {
+ continue;
+ }
+ int count = dv.docValueCount();
+ long[] arr = new long[count];
+ for (int i = 0; i < count; i++) {
+ arr[i] = dv.nextValue();
+ }
+ result.put(idByDoc.get(doc), arr);
+ }
+ }
+ return result;
+ }
+}