Apache Lucene open-source search software
Add dense indexing path for BinaryColumn (#16227)
Tim Brooks8a77a334aea5
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 978f361159f..4f23df5bf53 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -303,10 +303,12 @@ API Changes* GITHUB#16062: Add FlatFieldVectorsWriter.asKnnVectorValues to return directly a KnnVectorValues "view" overvectors to be written. (Lorenzo Dematte)-* GITHUB#16101 Add DictionaryColumn for SORTED / SORTED_SET docvalues using experimental column api (Tim Brooks)+* GITHUB#16101: Add DictionaryColumn for SORTED / SORTED_SET docvalues using experimental column api (Tim Brooks)* GITHUB#16104: Add public constructors and a public getSearchStrategy() to FloatVectorSimilarityQuery, ByteVectorSimilarityQuery and VectorSimilarityCollector that accept a KnnSearchStrategy. (Sasilekha R)+* GITHUB#16227: Add dense indexing path for BinaryColumn while using experimental column api (Tim Brooks)+New Features---------------------diff --git a/lucene/core/src/java/org/apache/lucene/document/column/BinaryColumn.java b/lucene/core/src/java/org/apache/lucene/document/column/BinaryColumn.javaindex b26c5b495fc..2c66c8be5f9 100644--- a/lucene/core/src/java/org/apache/lucene/document/column/BinaryColumn.java+++ b/lucene/core/src/java/org/apache/lucene/document/column/BinaryColumn.java@@ -54,4 +54,15 @@ public abstract class BinaryColumn extends Column {/** Returns a fresh tuple cursor starting at the beginning of the batch. */public abstract ObjectTupleCursor<BytesRef> tuples();++ /**+ * Returns a fresh values cursor iterating dense binary values for doc-ids {@code [0, numDocs)}.+ * Must be overridden when {@link Column#density()} is {@link Column.Density#DENSE DENSE}; the+ * default implementation throws {@link UnsupportedOperationException} and is never called for+ * {@link Column.Density#SPARSE SPARSE} columns.+ */+ public BytesRefValuesCursor values() {+ throw new UnsupportedOperationException(+ "values() requires density() == DENSE for column \"" + name() + "\"");+ }}diff --git a/lucene/core/src/java/org/apache/lucene/document/column/BytesRefValuesCursor.java b/lucene/core/src/java/org/apache/lucene/document/column/BytesRefValuesCursor.javanew file mode 100644index 00000000000..935d5dd1e3b--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/document/column/BytesRefValuesCursor.java@@ -0,0 +1,81 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.document.column;++import org.apache.lucene.util.BytesRef;++/**+ * A values cursor over a dense {@link BinaryColumn}. The cursor produces exactly {@link #size()}+ * values for consecutive batch-local doc-ids starting at 0, one per call to {@link #nextValue()}.+ *+ * <p>Implementations must throw an exception if {@link #nextValue()} is called more than {@link+ * #size()} times. The returned {@link BytesRef} is valid only until the next call to {@link+ * #nextValue()}.+ *+ * <p>Combined consumption across {@link #nextValue()} and {@link #fillPackedPoints} must not exceed+ * {@link #size()}.+ *+ * @lucene.experimental+ */+public abstract class BytesRefValuesCursor {++ private final int size;++ /**+ * Creates a cursor that will produce exactly {@code size} values, one per batch-local doc-id in+ * {@code [0, size)}. {@code size} is fixed for the cursor's lifetime and must equal the dense+ * column's {@code numDocs}.+ *+ * <p>Lucene's internal indexing paths will not consume past {@code size} across {@link+ * #nextValue()} and {@link #fillPackedPoints}. Defensive throws on overrun are still encouraged+ * to catch misuse from external callers.+ */+ protected BytesRefValuesCursor(int size) {+ this.size = size;+ }++ /** Total number of values this cursor will produce. */+ public final int size() {+ return size;+ }++ /**+ * Returns the next {@link BytesRef} value. Must not be called more than {@link #size()} times.+ */+ public abstract BytesRef nextValue();++ /**+ * Bulk-fill {@code length} fixed-width packed point records into {@code dst} starting at byte+ * {@code offset}, advancing the cursor by {@code length}. Each value must be exactly {@code+ * width} bytes and is passed through unchanged (no sortable encoding). Combined consumption+ * across {@link #nextValue()} and this method must not exceed {@link #size()}.+ *+ * <p>The default implementation calls {@link #nextValue()} per value and validates each length.+ * Override when the backing data is optimizable (e.g. a contiguous packed array); such overrides+ * take responsibility for the width contract.+ */+ public void fillPackedPoints(byte[] dst, int offset, int length, int width) {+ for (int i = 0; i < length; i++) {+ BytesRef v = nextValue();+ if (v.length != width) {+ throw new IllegalArgumentException(+ "dense point value has length=" + v.length + " but should be " + width);+ }+ System.arraycopy(v.bytes, v.offset, dst, offset + i * width, width);+ }+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.javaindex 3e7517fb04b..4d143340ed3 100644--- a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java+++ b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java@@ -44,6 +44,7 @@ import org.apache.lucene.document.KnnFloatVectorField;import org.apache.lucene.document.NumericDocValuesField;import org.apache.lucene.document.StoredValue;import org.apache.lucene.document.column.BinaryColumn;+import org.apache.lucene.document.column.BytesRefValuesCursor;import org.apache.lucene.document.column.Column;import org.apache.lucene.document.column.ColumnBatch;import org.apache.lucene.document.column.ColumnFieldAdapter;@@ -86,6 +87,8 @@ final class IndexingChain implements Accountable {final TermsHash termsHash;// Shared pool for doc-value termsfinal ByteBlockPool docValuesBytePool;+ // Shared scratch buffers for dense points encoding+ final SharedIndexingScratch sharedIndexingScratch;// Writes stored fieldsfinal StoredFieldsConsumer storedFieldsConsumer;final VectorValuesConsumer vectorValuesConsumer;@@ -155,6 +158,7 @@ final class IndexingChain implements Accountable {new FreqProxTermsWriter(intBlockAllocator, byteBlockAllocator, bytesUsed, termVectorsWriter);docValuesBytePool = new ByteBlockPool(byteBlockAllocator);+ sharedIndexingScratch = new SharedIndexingScratch(bytesUsed);if (indexWriterConfig.getParentField() != null) {this.parentField = new NumericDocValuesField(indexWriterConfig.getParentField(), -1);parentPf = getOrAddPerField(this.parentField.name());@@ -1082,6 +1086,12 @@ final class IndexingChain implements Accountable {throws IOException {final DocValuesType dvType = fieldType.docValuesType();final boolean hasPoints = fieldType.pointDimensionCount() != 0;++ if (column.density() == Column.Density.DENSE) {+ processDenseBinaryColumn(baseDocID, numDocs, column, pf, dvType, hasPoints);+ return;+ }+final PointValuesWriter pointWriter = hasPoints ? pf.pointValuesWriter : null;final ObjectTupleCursor<BytesRef> cursor = column.tuples();@@ -1143,6 +1153,51 @@ final class IndexingChain implements Accountable {}}+ private static void processDenseBinaryColumn(+ int baseDocID,+ int numDocs,+ BinaryColumn column,+ PerField pf,+ DocValuesType dvType,+ boolean hasPoints)+ throws IOException {+ // DV pass first: dense values cursor+ if (dvType != DocValuesType.NONE) {+ BytesRefValuesCursor dvCursor = column.values();+ ColumnValidation.checkDenseCount(column, dvCursor.size(), numDocs);+ switch (dvType) {+ case BINARY -> {+ BinaryDocValuesWriter writer = (BinaryDocValuesWriter) pf.docValuesWriter;+ for (int i = 0; i < dvCursor.size(); i++) {+ writer.addValue(baseDocID + i, dvCursor.nextValue());+ }+ }+ case SORTED -> {+ SortedDocValuesWriter writer = (SortedDocValuesWriter) pf.docValuesWriter;+ for (int i = 0; i < dvCursor.size(); i++) {+ writer.addValue(baseDocID + i, dvCursor.nextValue());+ }+ }+ case SORTED_SET -> {+ SortedSetDocValuesWriter writer = (SortedSetDocValuesWriter) pf.docValuesWriter;+ for (int i = 0; i < dvCursor.size(); i++) {+ writer.addValue(baseDocID + i, dvCursor.nextValue());+ }+ }+ // $CASES-OMITTED$+ default ->+ throw new IllegalArgumentException(+ "BinaryColumn \"" + column.name() + "\" has incompatible docValuesType: " + dvType);+ }+ }+ if (hasPoints) {+ // Points pass: fresh dense values cursor → bulk ND points add.+ BytesRefValuesCursor pc = column.values();+ ColumnValidation.checkDenseCount(column, pc.size(), numDocs);+ pf.pointValuesWriter.addDenseNDValues(baseDocID, pc);+ }+ }+private static void processDictionaryColumn(int baseDocID,int numDocs,@@ -1291,7 +1346,7 @@ final class IndexingChain implements Accountable {throw new AssertionError("unrecognized DocValues.Type: " + dvType);}if (fi.getPointDimensionCount() != 0) {- pf.pointValuesWriter = new PointValuesWriter(bytesUsed, fi);+ pf.pointValuesWriter = new PointValuesWriter(bytesUsed, fi, sharedIndexingScratch);}if (fi.getVectorDimension() != 0) {try {diff --git a/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.javaindex 5443d2919b7..4bff39999cc 100644--- a/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java+++ b/lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java@@ -20,12 +20,14 @@ import java.io.IOException;import org.apache.lucene.codecs.MutablePointTree;import org.apache.lucene.codecs.PointsReader;import org.apache.lucene.codecs.PointsWriter;+import org.apache.lucene.document.column.BytesRefValuesCursor;import org.apache.lucene.document.column.LongValuesCursor;import org.apache.lucene.store.DataOutput;import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.Counter;import org.apache.lucene.util.PagedBytes;+import org.apache.lucene.util.bkd.BKDConfig;/** Buffers up pending byte[][] value(s) per doc, then flushes when segment flushes. */class PointValuesWriter {@@ -33,26 +35,33 @@ class PointValuesWriter {private final PagedBytes bytes;private final DataOutput bytesOut;private final Counter iwBytesUsed;+ private final SharedIndexingScratch sharedScratch;private int[] docIDs;private int numPoints;private int numDocs;private int lastDocID = -1;private final int packedBytesLength;- /**- * Lazily-allocated 1 KB scratch used by the dense bulk paths to stage encoded points before- * pushing to {@link #bytesOut} via a single {@code writeBytes}.- */- private static final int POINTS_BUFFER_BYTES = 1024;-- private static final int POINTS_BUFFER_INT_VALUES = POINTS_BUFFER_BYTES / Integer.BYTES;- private static final int POINTS_BUFFER_LONG_VALUES = POINTS_BUFFER_BYTES / Long.BYTES;-- private byte[] densePointsBuffer;+ private static final int POINTS_BUFFER_INT_VALUES =+ SharedIndexingScratch.BYTES_SCRATCH_SIZE / Integer.BYTES;+ private static final int POINTS_BUFFER_LONG_VALUES =+ SharedIndexingScratch.BYTES_SCRATCH_SIZE / Long.BYTES;++ static {+ // If Lucene ever supports packed points > BYTES_SCRATCH_SIZE either the buffer needs to be+ // increased or a fallback indexing code-path bypassing the buffer needs to be added.+ assert SharedIndexingScratch.BYTES_SCRATCH_SIZE+ >= PointValues.MAX_NUM_BYTES * BKDConfig.MAX_DIMS+ : "BYTES_SCRATCH_SIZE="+ + SharedIndexingScratch.BYTES_SCRATCH_SIZE+ + " must be >= MAX_NUM_BYTES * MAX_DIMS="+ + (PointValues.MAX_NUM_BYTES * BKDConfig.MAX_DIMS);+ }- PointValuesWriter(Counter bytesUsed, FieldInfo fieldInfo) {+ PointValuesWriter(Counter bytesUsed, FieldInfo fieldInfo, SharedIndexingScratch sharedScratch) {this.fieldInfo = fieldInfo;this.iwBytesUsed = bytesUsed;+ this.sharedScratch = sharedScratch;this.bytes = new PagedBytes(12);bytesOut = bytes.getDataOutput();docIDs = new int[16];@@ -98,8 +107,8 @@ class PointValuesWriter {if (size == 0) {return;}- final long ramBefore = reserveDense1D(firstDocID, size);- final byte[] buffer = pointsDenseBuffer();+ final long ramBefore = reserveDenseRange(firstDocID, size);+ final byte[] buffer = sharedScratch.bytesScratch();int remaining = size;while (remaining > 0) {int chunk = Math.min(POINTS_BUFFER_INT_VALUES, remaining);@@ -107,7 +116,7 @@ class PointValuesWriter {bytesOut.writeBytes(buffer, 0, chunk * Integer.BYTES);remaining -= chunk;}- commitDense1D(firstDocID, size, ramBefore);+ commitDenseRange(firstDocID, size, ramBefore);}void addDense1DLongValues(int firstDocID, LongValuesCursor cursor) throws IOException {@@ -116,23 +125,35 @@ class PointValuesWriter {if (size == 0) {return;}- final long ramBefore = reserveDense1D(firstDocID, size);- final byte[] dense = pointsDenseBuffer();+ final long ramBefore = reserveDenseRange(firstDocID, size);+ final byte[] buffer = sharedScratch.bytesScratch();int remaining = size;while (remaining > 0) {int chunk = Math.min(POINTS_BUFFER_LONG_VALUES, remaining);- cursor.fillLongPoints(dense, 0, chunk);- bytesOut.writeBytes(dense, 0, chunk * Long.BYTES);+ cursor.fillLongPoints(buffer, 0, chunk);+ bytesOut.writeBytes(buffer, 0, chunk * Long.BYTES);remaining -= chunk;}- commitDense1D(firstDocID, size, ramBefore);+ commitDenseRange(firstDocID, size, ramBefore);}- private byte[] pointsDenseBuffer() {- if (densePointsBuffer == null) {- densePointsBuffer = new byte[POINTS_BUFFER_BYTES];+ void addDenseNDValues(int firstDocID, BytesRefValuesCursor cursor) throws IOException {+ final int size = cursor.size();+ if (size == 0) {+ return;+ }+ final long ramBefore = reserveDenseRange(firstDocID, size);+ final int width = packedBytesLength;+ final int perChunk = SharedIndexingScratch.BYTES_SCRATCH_SIZE / width;+ final byte[] buffer = sharedScratch.bytesScratch();+ int remaining = size;+ while (remaining > 0) {+ int chunk = Math.min(perChunk, remaining);+ cursor.fillPackedPoints(buffer, 0, chunk, width);+ bytesOut.writeBytes(buffer, 0, chunk * width);+ remaining -= chunk;}- return densePointsBuffer;+ commitDenseRange(firstDocID, size, ramBefore);}private void validate1DPacked(int byteWidth) {@@ -149,7 +170,7 @@ class PointValuesWriter {}}- private long reserveDense1D(int firstDocID, int size) {+ private long reserveDenseRange(int firstDocID, int size) {assert firstDocID > lastDocID: "firstDocID=" + firstDocID + " must be > lastDocID=" + lastDocID;final int oldLength = docIDs.length;@@ -163,7 +184,7 @@ class PointValuesWriter {return bytes.ramBytesUsed();}- private void commitDense1D(int firstDocID, int size, long ramBefore) {+ private void commitDenseRange(int firstDocID, int size, long ramBefore) {iwBytesUsed.addAndGet(bytes.ramBytesUsed() - ramBefore);numDocs += size;lastDocID = firstDocID + size - 1;diff --git a/lucene/core/src/java/org/apache/lucene/index/SharedIndexingScratch.java b/lucene/core/src/java/org/apache/lucene/index/SharedIndexingScratch.javanew file mode 100644index 00000000000..542a2acee0b--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/index/SharedIndexingScratch.java@@ -0,0 +1,55 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.index;++import org.apache.lucene.util.Counter;++/**+ * A holder of lazily-allocated, shared scratch buffers owned by {@link IndexingChain} and provided+ * to per-field writers that need transient buffers during indexing.+ *+ * <p>Because {@link IndexingChain} (and the {@link DocumentsWriterPerThread} it belongs to) indexes+ * documents single-threadedly, these buffers are safe to reuse across all writers within the same+ * chain. Callers must treat each buffer as transient scratch: fill it, drain it within the same+ * call, and not retain a reference across calls.+ */+final class SharedIndexingScratch {++ /** Size in bytes of the shared byte scratch buffer. */+ static final int BYTES_SCRATCH_SIZE = 4 * 1024;++ private final Counter bytesUsed;+ private byte[] bytesScratchBuffer;++ SharedIndexingScratch(Counter bytesUsed) {+ this.bytesUsed = bytesUsed;+ }++ /**+ * Returns the shared byte scratch buffer, allocating it on the first call and tracking its RAM+ * via the {@link Counter} supplied at construction.+ *+ * <p>Callers must treat the returned array as transient scratch.+ */+ byte[] bytesScratch() {+ if (bytesScratchBuffer == null) {+ bytesScratchBuffer = new byte[BYTES_SCRATCH_SIZE];+ bytesUsed.addAndGet(BYTES_SCRATCH_SIZE);+ }+ return bytesScratchBuffer;+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/document/column/ColumnBatchTestUtil.java b/lucene/core/src/test/org/apache/lucene/document/column/ColumnBatchTestUtil.javaindex e496603e90f..ab3fa496a1f 100644--- a/lucene/core/src/test/org/apache/lucene/document/column/ColumnBatchTestUtil.java+++ b/lucene/core/src/test/org/apache/lucene/document/column/ColumnBatchTestUtil.java@@ -147,6 +147,125 @@ public class ColumnBatchTestUtil {}}+ /** Dense {@link BinaryColumn} backed by a fixed {@link BytesRef} array, one value per doc. */+ public static class ArrayDenseBinaryColumn extends BinaryColumn {+ private final BytesRef[] values;++ public ArrayDenseBinaryColumn(String name, IndexableFieldType fieldType, BytesRef[] values) {+ super(name, fieldType, Density.DENSE);+ this.values = values;+ }++ @Override+ public ObjectTupleCursor<BytesRef> tuples() {+ return new ObjectTupleCursor<>() {+ int pos = -1;++ @Override+ public int nextDoc() {+ pos++;+ return pos < values.length ? pos : DocIdSetIterator.NO_MORE_DOCS;+ }++ @Override+ public BytesRef value() {+ return values[pos];+ }+ };+ }++ @Override+ public BytesRefValuesCursor values() {+ return new BytesRefValuesCursor(values.length) {+ int pos = 0;++ @Override+ public BytesRef nextValue() {+ if (pos >= values.length) {+ throw new IllegalStateException(+ "BytesRefValuesCursor exhausted: size=" + values.length);+ }+ return values[pos++];+ }+ };+ }+ }++ /**+ * Dense {@link BinaryColumn} backed by a single contiguous packed {@code byte[]} of {@code n *+ * width} bytes.+ */+ public static class ContiguousDenseBinaryColumn extends BinaryColumn {+ private final byte[] packed;+ private final int width;+ private final int n;++ /**+ * @param packed flat array of {@code n * width} bytes; {@code packed[i*width..(i+1)*width)} is+ * the value for batch-local doc-id {@code i}.+ * @param width number of bytes per value+ */+ public ContiguousDenseBinaryColumn(+ String name, IndexableFieldType fieldType, byte[] packed, int width) {+ super(name, fieldType, Density.DENSE);+ assert packed.length % width == 0 : "packed.length must be a multiple of width";+ this.packed = packed;+ this.width = width;+ this.n = packed.length / width;+ }++ @Override+ public ObjectTupleCursor<BytesRef> tuples() {+ return new ObjectTupleCursor<>() {+ int pos = -1;+ final BytesRef ref = new BytesRef(packed, 0, width);++ @Override+ public int nextDoc() {+ pos++;+ if (pos < n) {+ ref.offset = pos * width;+ return pos;+ }+ return DocIdSetIterator.NO_MORE_DOCS;+ }++ @Override+ public BytesRef value() {+ return ref;+ }+ };+ }++ @Override+ public BytesRefValuesCursor values() {+ return new BytesRefValuesCursor(n) {+ int consumed = 0;+ final BytesRef ref = new BytesRef(packed, 0, width);++ @Override+ public BytesRef nextValue() {+ if (consumed >= n) {+ throw new IllegalStateException("BytesRefValuesCursor exhausted: size=" + n);+ }+ ref.offset = consumed * width;+ consumed++;+ return ref;+ }++ /** Single bulk copy — exercises the override path. */+ @Override+ public void fillPackedPoints(byte[] dst, int offset, int length, int width) {+ if (consumed + length > n) {+ throw new IllegalStateException("BytesRefValuesCursor exhausted: size=" + n);+ }+ System.arraycopy(packed, consumed * width, dst, offset, length * width);+ consumed += length;+ }+ };+ }+ }+/** Dense {@link LongColumn} with an optional bulk values cursor. */public static class ArrayDenseLongColumn extends LongColumn {private final long[] values;diff --git a/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchBinaryColumn.java b/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchBinaryColumn.javaindex 66579f427bb..de51ecf6127 100644--- a/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchBinaryColumn.java+++ b/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchBinaryColumn.java@@ -17,11 +17,14 @@package org.apache.lucene.document.column;import static org.apache.lucene.document.column.ColumnBatchTestUtil.ArrayBinaryColumn;+import static org.apache.lucene.document.column.ColumnBatchTestUtil.ArrayDenseBinaryColumn;import static org.apache.lucene.document.column.ColumnBatchTestUtil.ArrayLongColumn;+import static org.apache.lucene.document.column.ColumnBatchTestUtil.ContiguousDenseBinaryColumn;import static org.apache.lucene.document.column.ColumnBatchTestUtil.simpleBatch;import java.io.IOException;import org.apache.lucene.document.BinaryDocValuesField;+import org.apache.lucene.document.BinaryPoint;import org.apache.lucene.document.Document;import org.apache.lucene.document.FieldType;import org.apache.lucene.document.IntPoint;@@ -50,6 +53,7 @@ import org.apache.lucene.store.Directory;import org.apache.lucene.tests.analysis.MockAnalyzer;import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.NumericUtils;/** Tests for {@link BinaryColumn} batch indexing. */public class TestColumnBatchBinaryColumn extends LuceneTestCase {@@ -693,4 +697,208 @@ public class TestColumnBatchBinaryColumn extends LuceneTestCase {w.close();dir.close();}++ public void testDenseBinaryPoints() throws IOException {+ Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());++ // 1-D, 4-byte binary points (sort-encoded ints).+ FieldType type = new FieldType();+ type.setDimensions(1, Integer.BYTES);+ type.freeze();++ int[] rawValues = {-100, 0, 1, 42, 999};+ BytesRef[] encoded = new BytesRef[rawValues.length];+ for (int i = 0; i < rawValues.length; i++) {+ byte[] b = new byte[Integer.BYTES];+ NumericUtils.intToSortableBytes(rawValues[i], b, 0);+ encoded[i] = new BytesRef(b);+ }++ w.addBatch(simpleBatch(rawValues.length, new ArrayDenseBinaryColumn("p", type, encoded)));++ DirectoryReader r = DirectoryReader.open(w);+ IndexSearcher searcher = new IndexSearcher(r);++ // All docs match a full-range query.+ byte[] minB = new byte[Integer.BYTES];+ byte[] maxB = new byte[Integer.BYTES];+ NumericUtils.intToSortableBytes(Integer.MIN_VALUE, minB, 0);+ NumericUtils.intToSortableBytes(Integer.MAX_VALUE, maxB, 0);+ assertEquals(rawValues.length, searcher.count(BinaryPoint.newRangeQuery("p", minB, maxB)));++ // Exact queries for first, last, and a middle value.+ for (int raw : new int[] {rawValues[0], rawValues[rawValues.length - 1], rawValues[2]}) {+ byte[] exact = new byte[Integer.BYTES];+ NumericUtils.intToSortableBytes(raw, exact, 0);+ assertEquals(1, searcher.count(BinaryPoint.newExactQuery("p", exact)));+ }++ // A value that was not indexed should match nothing.+ byte[] absent = new byte[Integer.BYTES];+ NumericUtils.intToSortableBytes(-1, absent, 0);+ assertEquals(0, searcher.count(BinaryPoint.newExactQuery("p", absent)));++ r.close();+ w.close();+ dir.close();+ }++ public void testDenseBinaryPointsWithSortedDocValues() throws IOException {+ Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());++ // SORTED DV + 1-D 4-byte points. Same BytesRef serves both.+ FieldType type = new FieldType();+ type.setDocValuesType(org.apache.lucene.index.DocValuesType.SORTED);+ type.setDimensions(1, Integer.BYTES);+ type.freeze();++ // 4 docs; sort-encoded ints are used for both DV and points.+ final int n = 4;+ BytesRef[] encoded = new BytesRef[n];+ for (int i = 0; i < n; i++) {+ byte[] b = new byte[Integer.BYTES];+ NumericUtils.intToSortableBytes(i * 10, b, 0);+ encoded[i] = new BytesRef(b);+ }++ w.addBatch(simpleBatch(n, new ArrayDenseBinaryColumn("sdv", type, encoded)));++ DirectoryReader r = DirectoryReader.open(w);+ LeafReader leaf = getOnlyLeafReader(r);+ IndexSearcher searcher = new IndexSearcher(leaf);++ // Verify sorted DV: all 4 docs should have DV (4 distinct values).+ SortedDocValues dv = leaf.getSortedDocValues("sdv");+ assertNotNull(dv);+ assertEquals(n, dv.getValueCount());+ for (int i = 0; i < n; i++) {+ assertEquals(i, dv.nextDoc());+ assertEquals(encoded[i], dv.lookupOrd(dv.ordValue()));+ }+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, dv.nextDoc());++ // Verify points: exact query for each encoded value returns exactly 1 doc.+ for (int i = 0; i < n; i++) {+ assertEquals(1, searcher.count(BinaryPoint.newExactQuery("sdv", encoded[i].bytes.clone())));+ }+ // Range query spanning all values.+ byte[] minB = new byte[Integer.BYTES];+ byte[] maxB = new byte[Integer.BYTES];+ NumericUtils.intToSortableBytes(Integer.MIN_VALUE, minB, 0);+ NumericUtils.intToSortableBytes(Integer.MAX_VALUE, maxB, 0);+ assertEquals(n, searcher.count(BinaryPoint.newRangeQuery("sdv", minB, maxB)));++ r.close();+ w.close();+ dir.close();+ }++ public void testDenseBinaryPointsLargeBatch() throws IOException {+ Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());++ // 1-D, 8-bytes-per-dim binary points: packedLen = 8, chunk = 4096/8 = 512.+ final int bytesPerDim = Long.BYTES;+ FieldType type = new FieldType();+ type.setDimensions(1, bytesPerDim);+ type.freeze();++ // n = 1100 > 2 * 512 → loop runs at least 3 times.+ final int n = 1100;+ BytesRef[] encoded = new BytesRef[n];+ for (int i = 0; i < n; i++) {+ byte[] b = new byte[bytesPerDim];+ NumericUtils.longToSortableBytes(i * 1_000_000L - 100L, b, 0);+ encoded[i] = new BytesRef(b);+ }++ w.addBatch(simpleBatch(n, new ArrayDenseBinaryColumn("p1d8", type, encoded)));++ DirectoryReader r = DirectoryReader.open(w);+ IndexSearcher searcher = new IndexSearcher(r);++ // All docs indexed.+ byte[] minB = new byte[bytesPerDim];+ byte[] maxB = new byte[bytesPerDim];+ NumericUtils.longToSortableBytes(Long.MIN_VALUE, minB, 0);+ NumericUtils.longToSortableBytes(Long.MAX_VALUE, maxB, 0);+ assertEquals(n, searcher.count(BinaryPoint.newRangeQuery("p1d8", minB, maxB)));++ // Values at and around the chunk boundary (index 511 / 512) round-trip correctly.+ assertEquals(1, searcher.count(BinaryPoint.newExactQuery("p1d8", encoded[511].bytes.clone())));+ assertEquals(1, searcher.count(BinaryPoint.newExactQuery("p1d8", encoded[512].bytes.clone())));++ // A value that was not indexed.+ byte[] absent = new byte[bytesPerDim];+ absent[0] = (byte) 0xFF; // sorts above everything we indexed+ assertEquals(0, searcher.count(BinaryPoint.newExactQuery("p1d8", absent)));++ r.close();+ w.close();+ dir.close();+ }++ public void testDenseBinaryPointsContiguousFill() throws IOException {+ final int bytesPerDim = Long.BYTES; // packedLen = 8, perChunk = 512+ final int n = 600; // spans two chunk iterations (512 + 88)++ FieldType type = new FieldType();+ type.setDimensions(1, bytesPerDim);+ type.freeze();++ // Build both representations from the same encoded values.+ BytesRef[] perValueArray = new BytesRef[n];+ byte[] packedFlat = new byte[n * bytesPerDim];+ for (int i = 0; i < n; i++) {+ byte[] b = new byte[bytesPerDim];+ NumericUtils.longToSortableBytes(i * 500L, b, 0);+ perValueArray[i] = new BytesRef(b);+ System.arraycopy(b, 0, packedFlat, i * bytesPerDim, bytesPerDim);+ }++ // Index via the default per-value path (ArrayDenseBinaryColumn).+ Directory dirDefault = newDirectory();+ IndexWriter wDefault = new IndexWriter(dirDefault, newIndexWriterConfig());+ wDefault.addBatch(simpleBatch(n, new ArrayDenseBinaryColumn("p", type, perValueArray)));+ DirectoryReader rDefault = DirectoryReader.open(wDefault);+ IndexSearcher searcherDefault = new IndexSearcher(rDefault);++ // Index via the contiguous bulk-copy override path (ContiguousDenseBinaryColumn).+ Directory dirBulk = newDirectory();+ IndexWriter wBulk = new IndexWriter(dirBulk, newIndexWriterConfig());+ wBulk.addBatch(+ simpleBatch(n, new ContiguousDenseBinaryColumn("p", type, packedFlat, bytesPerDim)));+ DirectoryReader rBulk = DirectoryReader.open(wBulk);+ IndexSearcher searcherBulk = new IndexSearcher(rBulk);++ // Both indexes must return the same results for a full range query.+ byte[] minB = new byte[bytesPerDim];+ byte[] maxB = new byte[bytesPerDim];+ NumericUtils.longToSortableBytes(Long.MIN_VALUE, minB, 0);+ NumericUtils.longToSortableBytes(Long.MAX_VALUE, maxB, 0);+ assertEquals(n, searcherDefault.count(BinaryPoint.newRangeQuery("p", minB, maxB)));+ assertEquals(n, searcherBulk.count(BinaryPoint.newRangeQuery("p", minB, maxB)));++ // Exact queries for a sample of values, including the chunk boundary (index 511/512).+ for (int idx : new int[] {0, 1, 255, 511, 512, 513, n - 1}) {+ byte[] exact = perValueArray[idx].bytes.clone();+ assertEquals(+ "default path mismatch at idx=" + idx,+ 1,+ searcherDefault.count(BinaryPoint.newExactQuery("p", exact)));+ assertEquals(+ "bulk override path mismatch at idx=" + idx,+ 1,+ searcherBulk.count(BinaryPoint.newExactQuery("p", exact)));+ }++ rDefault.close();+ wDefault.close();+ dirDefault.close();+ rBulk.close();+ wBulk.close();+ dirBulk.close();+ }}diff --git a/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchLongColumn.java b/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchLongColumn.javaindex b309a016dfb..fe0664a84ad 100644--- a/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchLongColumn.java+++ b/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchLongColumn.java@@ -1178,7 +1178,7 @@ public class TestColumnBatchLongColumn extends LuceneTestCase {}/**- * Dense long batch larger than {@code POINTS_BUFFER_LONG_VALUES} (128) so the chunked encoding+ * Dense long batch larger than {@code POINTS_BUFFER_LONG_VALUES} (512) so the chunked encoding* loop in {@code PointValuesWriter#addDense1DLongValues} runs more than once.*/public void testDensePointsLongLargeBatch() throws IOException {@@ -1189,7 +1189,7 @@ public class TestColumnBatchLongColumn extends LuceneTestCase {type.setDimensions(1, Long.BYTES);type.freeze();- final int n = 350; // > 2 * 128 chunks+ final int n = 1100; // > 2 * 512 chunkslong[] values = new long[n];for (int i = 0; i < n; i++) {values[i] = i * 1_000_000L - 100L; // mix of negative and positive@@ -1201,8 +1201,8 @@ public class TestColumnBatchLongColumn extends LuceneTestCase {assertEquals(n, searcher.count(LongPoint.newRangeQuery("v", Long.MIN_VALUE, Long.MAX_VALUE)));assertEquals(1, searcher.count(LongPoint.newExactQuery("v", values[0])));assertEquals(1, searcher.count(LongPoint.newExactQuery("v", values[n - 1])));- assertEquals(1, searcher.count(LongPoint.newExactQuery("v", values[127]))); // chunk boundary- assertEquals(1, searcher.count(LongPoint.newExactQuery("v", values[128])));+ assertEquals(1, searcher.count(LongPoint.newExactQuery("v", values[511]))); // chunk boundary+ assertEquals(1, searcher.count(LongPoint.newExactQuery("v", values[512])));assertEquals(0, searcher.count(LongPoint.newExactQuery("v", -1L)));assertEquals(n, searcher.count(LongPoint.newRangeQuery("v", values[0], values[n - 1])));