Apache Lucene open-source search software
Add back maxConn & beamWidth HNSW codec ctor (#12728)
Benjamin Trent2a8d187a99e9
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsFormat.javaindex 03a6a4dbd62..038b75e4c48 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsFormat.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsFormat.java@@ -191,6 +191,16 @@ public final class Lucene99HnswVectorsFormat extends KnnVectorsFormat {** @param maxConn the maximum number of connections to a node in the HNSW graph* @param beamWidth the size of the queue maintained during graph construction.+ */+ public Lucene99HnswVectorsFormat(int maxConn, int beamWidth) {+ this(maxConn, beamWidth, null);+ }++ /**+ * Constructs a format using the given graph construction parameters and scalar quantization.+ *+ * @param maxConn the maximum number of connections to a node in the HNSW graph+ * @param beamWidth the size of the queue maintained during graph construction.* @param scalarQuantize the scalar quantization format* @param numMergeWorkers number of workers (threads) that will be used when doing merge. If* larger than 1, a non-null {@link ExecutorService} must be passed as mergeExecdiff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.javaindex 2961dbe1f1e..c0485810fc8 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsWriter.java@@ -234,7 +234,8 @@ public final class Lucene99ScalarQuantizedVectorsWriter implements Accountable {MergedQuantizedVectorValues byteVectorValues =MergedQuantizedVectorValues.mergeQuantizedByteVectorValues(fieldInfo, mergeState, mergedQuantizationState);- writeQuantizedVectorData(tempQuantizedVectorData, byteVectorValues);+ DocsWithFieldSet docsWithField =+ writeQuantizedVectorData(tempQuantizedVectorData, byteVectorValues);CodecUtil.writeFooter(tempQuantizedVectorData);IOUtils.close(tempQuantizedVectorData);quantizationDataInput =@@ -254,7 +255,9 @@ public final class Lucene99ScalarQuantizedVectorsWriter implements Accountable {fieldInfo.getVectorSimilarityFunction(),mergedQuantizationState,new OffHeapQuantizedByteVectorValues.DenseOffHeapVectorValues(- fieldInfo.getVectorDimension(), byteVectorValues.size(), quantizationDataInput)));+ fieldInfo.getVectorDimension(),+ docsWithField.cardinality(),+ quantizationDataInput)));} finally {if (success == false) {IOUtils.closeWhileHandlingException(quantizationDataInput);diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswVectorsFormat.javaindex aa9c9f109e0..8f2fdd2c75f 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswVectorsFormat.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswVectorsFormat.java@@ -33,7 +33,7 @@ public class TestLucene99HnswVectorsFormat extends BaseKnnVectorsFormatTestCasenew FilterCodec("foo", Codec.getDefault()) {@Overridepublic KnnVectorsFormat knnVectorsFormat() {- return new Lucene99HnswVectorsFormat(10, 20, null);+ return new Lucene99HnswVectorsFormat(10, 20);}};String expectedString =@@ -42,13 +42,11 @@ public class TestLucene99HnswVectorsFormat extends BaseKnnVectorsFormatTestCase}public void testLimits() {- expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(-1, 20, null));- expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(0, 20, null));- expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(20, 0, null));- expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(20, -1, null));- expectThrows(- IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(512 + 1, 20, null));- expectThrows(- IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(20, 3201, null));+ expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(-1, 20));+ expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(0, 20));+ expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(20, 0));+ expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(20, -1));+ expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(512 + 1, 20));+ expectThrows(IllegalArgumentException.class, () -> new Lucene99HnswVectorsFormat(20, 3201));}}diff --git a/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldKnnVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldKnnVectorsFormat.javaindex c578ae85c55..c51f51f5f11 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldKnnVectorsFormat.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldKnnVectorsFormat.java@@ -170,8 +170,8 @@ public class TestPerFieldKnnVectorsFormat extends BaseKnnVectorsFormatTestCase {try (Directory directory = newDirectory()) {IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));KnnVectorsFormat format1 =- new KnnVectorsFormatMaxDims32(new Lucene99HnswVectorsFormat(16, 100, null));- KnnVectorsFormat format2 = new Lucene99HnswVectorsFormat(16, 100, null);+ new KnnVectorsFormatMaxDims32(new Lucene99HnswVectorsFormat(16, 100));+ KnnVectorsFormat format2 = new Lucene99HnswVectorsFormat(16, 100);iwc.setCodec(new AssertingCodec() {@Overridediff --git a/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java b/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.javaindex 368f6f9be10..32f325970ab 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java@@ -113,8 +113,7 @@ public class TestKnnGraph extends LuceneTestCase {return new PerFieldKnnVectorsFormat() {@Overridepublic KnnVectorsFormat getKnnVectorsFormatForField(String field) {- return new Lucene99HnswVectorsFormat(- M, HnswGraphBuilder.DEFAULT_BEAM_WIDTH, null);+ return new Lucene99HnswVectorsFormat(M, HnswGraphBuilder.DEFAULT_BEAM_WIDTH);}};}diff --git a/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java b/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.javaindex ebb10ad5b5f..22670e19fa6 100644--- a/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java+++ b/lucene/core/src/test/org/apache/lucene/util/hnsw/HnswGraphTestCase.java@@ -165,7 +165,7 @@ abstract class HnswGraphTestCase<T> extends LuceneTestCase {return new PerFieldKnnVectorsFormat() {@Overridepublic KnnVectorsFormat getKnnVectorsFormatForField(String field) {- return new Lucene99HnswVectorsFormat(M, beamWidth, null);+ return new Lucene99HnswVectorsFormat(M, beamWidth);}};}@@ -237,7 +237,7 @@ abstract class HnswGraphTestCase<T> extends LuceneTestCase {return new PerFieldKnnVectorsFormat() {@Overridepublic KnnVectorsFormat getKnnVectorsFormatForField(String field) {- return new Lucene99HnswVectorsFormat(M, beamWidth, null);+ return new Lucene99HnswVectorsFormat(M, beamWidth);}};}@@ -298,7 +298,7 @@ abstract class HnswGraphTestCase<T> extends LuceneTestCase {return new PerFieldKnnVectorsFormat() {@Overridepublic KnnVectorsFormat getKnnVectorsFormatForField(String field) {- return new Lucene99HnswVectorsFormat(M, beamWidth, null);+ return new Lucene99HnswVectorsFormat(M, beamWidth);}};}@@ -312,7 +312,7 @@ abstract class HnswGraphTestCase<T> extends LuceneTestCase {return new PerFieldKnnVectorsFormat() {@Overridepublic KnnVectorsFormat getKnnVectorsFormatForField(String field) {- return new Lucene99HnswVectorsFormat(M, beamWidth, null);+ return new Lucene99HnswVectorsFormat(M, beamWidth);}};}diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/vector/ConfigurableMCodec.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/vector/ConfigurableMCodec.javaindex 4cd473cc01f..11aa4aced97 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/vector/ConfigurableMCodec.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/vector/ConfigurableMCodec.java@@ -32,12 +32,12 @@ public class ConfigurableMCodec extends FilterCodec {public ConfigurableMCodec() {super("ConfigurableMCodec", TestUtil.getDefaultCodec());- knnVectorsFormat = new Lucene99HnswVectorsFormat(128, 100, null);+ knnVectorsFormat = new Lucene99HnswVectorsFormat(128, 100);}public ConfigurableMCodec(int maxConn) {super("ConfigurableMCodec", TestUtil.getDefaultCodec());- knnVectorsFormat = new Lucene99HnswVectorsFormat(maxConn, 100, null);+ knnVectorsFormat = new Lucene99HnswVectorsFormat(maxConn, 100);}@Override