Apache Lucene open-source search software
Randomize KnnVector codec params in RandomCodec; addresses gh-14047 (#14049)
Michael Sokolov6b0112cdee28
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 5bb515d3cba..a76302c5742 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -24,7 +24,8 @@ OptimizationsBug Fixes----------------------(No changes)+* GITHUB#14049: Randomize KNN codec params in RandomCodec. Fixes scalar quantization div-by-zero+ when all values are identical. (Mike Sokolov)Other---------------------diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBasicBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBasicBackwardsCompatibility.javaindex 262567f9f76..a93c5c7617a 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBasicBackwardsCompatibility.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBasicBackwardsCompatibility.java@@ -541,6 +541,7 @@ public class TestBasicBackwardsCompatibility extends BackwardsCompatibilityTestBnew IndexWriter(dir,newIndexWriterConfig(new MockAnalyzer(random))+ .setCodec(TestUtil.getDefaultCodec()).setOpenMode(IndexWriterConfig.OpenMode.APPEND).setMergePolicy(newLogMergePolicy()));// add 10 docs@@ -579,6 +580,7 @@ public class TestBasicBackwardsCompatibility extends BackwardsCompatibilityTestBnew IndexWriter(dir,newIndexWriterConfig(new MockAnalyzer(random))+ .setCodec(TestUtil.getDefaultCodec()).setOpenMode(IndexWriterConfig.OpenMode.APPEND).setMergePolicy(newLogMergePolicy()));writer.forceMerge(1);@@ -630,6 +632,7 @@ public class TestBasicBackwardsCompatibility extends BackwardsCompatibilityTestBnew IndexWriter(dir,newIndexWriterConfig(new MockAnalyzer(random))+ .setCodec(TestUtil.getDefaultCodec()).setOpenMode(IndexWriterConfig.OpenMode.APPEND));writer.forceMerge(1);writer.close();diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.javaindex a4770f01f46..8fc417e22f0 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java@@ -93,6 +93,7 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {VectorSimilarityFunction sim,float constMultiplier,QuantizedByteVectorValues values) {+ checkDimensions(targetBytes.length, values.dimension());return switch (sim) {case EUCLIDEAN -> new Euclidean(values, constMultiplier, targetBytes);case COSINE, DOT_PRODUCT ->@@ -112,6 +113,13 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {};}+ static void checkDimensions(int queryLen, int fieldLen) {+ if (queryLen != fieldLen) {+ throw new IllegalArgumentException(+ "vector query dimension: " + queryLen + " differs from field dimension: " + fieldLen);+ }+ }+private static RandomVectorScorer.AbstractRandomVectorScorer dotProductFactory(byte[] targetBytes,float offsetCorrection,diff --git a/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java b/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.javaindex 3f7bcf6c5c4..34fb380a183 100644--- a/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java+++ b/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java@@ -97,12 +97,18 @@ public class ScalarQuantizer {}assert maxQuantile >= minQuantile;assert bits > 0 && bits <= 8;- this.minQuantile = minQuantile;- this.maxQuantile = maxQuantile;this.bits = bits;final float divisor = (float) ((1 << bits) - 1);- this.scale = divisor / (maxQuantile - minQuantile);- this.alpha = (maxQuantile - minQuantile) / divisor;+ if (minQuantile == maxQuantile) {+ // avoid divide-by-zero with an arbitrary but plausible choice (leads to alpha = scale = 1)+ this.minQuantile = minQuantile - divisor;+ this.maxQuantile = maxQuantile + divisor;+ } else {+ this.minQuantile = minQuantile;+ this.maxQuantile = maxQuantile;+ }+ this.scale = divisor / (this.maxQuantile - this.minQuantile);+ this.alpha = (this.maxQuantile - this.minQuantile) / divisor;}/**diff --git a/lucene/core/src/test/org/apache/lucene/search/BaseKnnVectorQueryTestCase.java b/lucene/core/src/test/org/apache/lucene/search/BaseKnnVectorQueryTestCase.javaindex 4abd955eeb6..1700a1dbae8 100644--- a/lucene/core/src/test/org/apache/lucene/search/BaseKnnVectorQueryTestCase.java+++ b/lucene/core/src/test/org/apache/lucene/search/BaseKnnVectorQueryTestCase.java@@ -19,7 +19,6 @@ package org.apache.lucene.search;import static com.carrotsearch.randomizedtesting.RandomizedTest.frequently;import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean;import static com.carrotsearch.randomizedtesting.RandomizedTest.randomIntBetween;-import static org.apache.lucene.index.VectorSimilarityFunction.COSINE;import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;import java.io.IOException;@@ -61,6 +60,8 @@ import org.apache.lucene.util.FixedBitSet;/** Test cases for AbstractKnnVectorQuery objects. */abstract class BaseKnnVectorQueryTestCase extends LuceneTestCase {+ // handle quantization noise+ static final float EPSILON = 0.001f;abstract AbstractKnnVectorQuery getKnnVectorQuery(String field, float[] query, int k, Query queryFilter);@@ -86,6 +87,10 @@ abstract class BaseKnnVectorQueryTestCase extends LuceneTestCase {return LuceneTestCase.newDirectory(random());}+ protected IndexWriterConfig configStandardCodec() throws IOException {+ return new IndexWriterConfig().setCodec(TestUtil.getDefaultCodec());+ }+public void testEquals() {AbstractKnnVectorQuery q1 = getKnnVectorQuery("f1", new float[] {0, 1}, 10);Query filter1 = new TermQuery(new Term("id", "id1"));@@ -317,66 +322,58 @@ abstract class BaseKnnVectorQueryTestCase extends LuceneTestCase {}public void testScoreCosine() throws IOException {- try (Directory d = newDirectoryForTest()) {- try (IndexWriter w = new IndexWriter(d, new IndexWriterConfig())) {- for (int j = 1; j <= 5; j++) {- Document doc = new Document();- doc.add(getKnnVectorField("field", new float[] {j, j * j}, COSINE));- w.addDocument(doc);- }- }- try (IndexReader reader = DirectoryReader.open(d)) {- assertEquals(1, reader.leaves().size());- IndexSearcher searcher = new IndexSearcher(reader);- AbstractKnnVectorQuery query = getKnnVectorQuery("field", new float[] {2, 3}, 3);- Query rewritten = query.rewrite(searcher);- Weight weight = searcher.createWeight(rewritten, ScoreMode.COMPLETE, 1);- Scorer scorer = weight.scorer(reader.leaves().get(0));-- // prior to advancing, score is undefined- assertEquals(-1, scorer.docID());- expectThrows(ArrayIndexOutOfBoundsException.class, scorer::score);-- /* score0 = ((2,3) * (1, 1) = 5) / (||2, 3|| * ||1, 1|| = sqrt(26)), then- * normalized by (1 + x) /2.- */- float score0 =- (float) ((1 + (2 * 1 + 3 * 1) / Math.sqrt((2 * 2 + 3 * 3) * (1 * 1 + 1 * 1))) / 2);-- /* score1 = ((2,3) * (2, 4) = 16) / (||2, 3|| * ||2, 4|| = sqrt(260)), then- * normalized by (1 + x) /2- */- float score1 =- (float) ((1 + (2 * 2 + 3 * 4) / Math.sqrt((2 * 2 + 3 * 3) * (2 * 2 + 4 * 4))) / 2);-- // doc 1 happens to have the maximum score- assertEquals(score1, scorer.getMaxScore(2), 0.0001);- assertEquals(score1, scorer.getMaxScore(Integer.MAX_VALUE), 0.0001);-- DocIdSetIterator it = scorer.iterator();- assertEquals(3, it.cost());- assertEquals(0, it.nextDoc());- // doc 0 has (1, 1)- assertEquals(score0, scorer.score(), 0.0001);- assertEquals(1, it.advance(1));- assertEquals(score1, scorer.score(), 0.0001);-- // since topK was 3- assertEquals(NO_MORE_DOCS, it.advance(4));- expectThrows(ArrayIndexOutOfBoundsException.class, scorer::score);- }+ float[][] vectors = new float[5][];+ for (int j = 1; j <= 5; j++) {+ vectors[j - 1] = new float[] {j, j * j};+ }+ try (Directory d = getStableIndexStore("field", VectorSimilarityFunction.COSINE, vectors);+ IndexReader reader = DirectoryReader.open(d)) {+ assertEquals(1, reader.leaves().size());+ IndexSearcher searcher = new IndexSearcher(reader);+ AbstractKnnVectorQuery query = getKnnVectorQuery("field", new float[] {2, 3}, 3);+ Query rewritten = query.rewrite(searcher);+ Weight weight = searcher.createWeight(rewritten, ScoreMode.COMPLETE, 1);+ Scorer scorer = weight.scorer(reader.leaves().get(0));++ // prior to advancing, score is undefined+ assertEquals(-1, scorer.docID());+ expectThrows(ArrayIndexOutOfBoundsException.class, scorer::score);++ /* score0 = ((2,3) * (1, 1) = 5) / (||2, 3|| * ||1, 1|| = sqrt(26)), then+ * normalized by (1 + x) /2.+ */+ float score0 =+ (float) ((1 + (2 * 1 + 3 * 1) / Math.sqrt((2 * 2 + 3 * 3) * (1 * 1 + 1 * 1))) / 2);++ /* score1 = ((2,3) * (2, 4) = 16) / (||2, 3|| * ||2, 4|| = sqrt(260)), then+ * normalized by (1 + x) /2+ */+ float score1 =+ (float) ((1 + (2 * 2 + 3 * 4) / Math.sqrt((2 * 2 + 3 * 3) * (2 * 2 + 4 * 4))) / 2);++ // doc 1 happens to have the maximum score+ assertEquals(score1, scorer.getMaxScore(2), 0.0001);+ assertEquals(score1, scorer.getMaxScore(Integer.MAX_VALUE), 0.0001);++ DocIdSetIterator it = scorer.iterator();+ assertEquals(3, it.cost());+ assertEquals(0, it.nextDoc());+ // doc 0 has (1, 1)+ assertEquals(score0, scorer.score(), 0.0001);+ assertEquals(1, it.advance(1));+ assertEquals(score1, scorer.score(), 0.0001);++ // since topK was 3+ assertEquals(NO_MORE_DOCS, it.advance(4));+ expectThrows(ArrayIndexOutOfBoundsException.class, scorer::score);}}public void testScoreMIP() throws IOException {- try (Directory indexStore =- getIndexStore(- "field",- VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT,- new float[] {0, 1},- new float[] {1, 2},- new float[] {0, 0});- IndexReader reader = DirectoryReader.open(indexStore)) {+ float[][] vectors = {{0, 1}, {1, 2}, {0, 0}};+ try (Directory d =+ getStableIndexStore("field", VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT, vectors);+ IndexReader reader = DirectoryReader.open(d)) {IndexSearcher searcher = newSearcher(reader);AbstractKnnVectorQuery kvq = getKnnVectorQuery("field", new float[] {0, -1}, 10);assertMatches(searcher, kvq, 3);@@ -405,7 +402,8 @@ abstract class BaseKnnVectorQueryTestCase extends LuceneTestCase {AbstractKnnVectorQuery query = getKnnVectorQuery("field", new float[] {2, 3}, 3);Explanation matched = searcher.explain(query, 2);assertTrue(matched.isMatch());- assertEquals(1 / 2f, matched.getValue());+ // scores vary widely due to quantization+ assertEquals(1 / 2f, matched.getValue().doubleValue(), 0.5);assertEquals(0, matched.getDetails().length);assertEquals("within top 3 docs", matched.getDescription());@@ -431,9 +429,10 @@ abstract class BaseKnnVectorQueryTestCase extends LuceneTestCase {try (IndexReader reader = DirectoryReader.open(d)) {IndexSearcher searcher = new IndexSearcher(reader);AbstractKnnVectorQuery query = getKnnVectorQuery("field", new float[] {2, 3}, 3);- Explanation matched = searcher.explain(query, 2);+ Explanation matched = searcher.explain(query, 2); // (2, 2)assertTrue(matched.isMatch());- assertEquals(1 / 2f, matched.getValue());+ // scores vary widely due to quantization+ assertEquals(1 / 2f, matched.getValue().doubleValue(), 0.5);assertEquals(0, matched.getDetails().length);assertEquals("within top 3 docs", matched.getDescription());@@ -453,7 +452,7 @@ abstract class BaseKnnVectorQueryTestCase extends LuceneTestCase {* randomly fail to find one).*/try (Directory d = newDirectoryForTest()) {- try (IndexWriter w = new IndexWriter(d, new IndexWriterConfig())) {+ try (IndexWriter w = new IndexWriter(d, configStandardCodec())) {int r = 0;for (int i = 0; i < 5; i++) {for (int j = 0; j < 5; j++) {@@ -532,7 +531,7 @@ abstract class BaseKnnVectorQueryTestCase extends LuceneTestCase {// visitedLimit. This is fine since the test targets AbstractKnnVectorQuery logic, not the kNN// format// implementation.- IndexWriterConfig iwc = new IndexWriterConfig().setCodec(TestUtil.getDefaultCodec());+ IndexWriterConfig iwc = configStandardCodec();RandomIndexWriter w = new RandomIndexWriter(random(), d, iwc);for (int i = 0; i < numDocs; i++) {Document doc = new Document();@@ -618,7 +617,7 @@ abstract class BaseKnnVectorQueryTestCase extends LuceneTestCase {// visitedLimit. This is fine since the test targets AbstractKnnVectorQuery logic, not the kNN// format// implementation.- IndexWriterConfig iwc = new IndexWriterConfig().setCodec(TestUtil.getDefaultCodec());+ IndexWriterConfig iwc = configStandardCodec();IndexWriter w = new IndexWriter(d, iwc);float[] vector = randomVector(dimension);for (int i = 0; i < numDocs; i++) {@@ -933,11 +932,17 @@ abstract class BaseKnnVectorQueryTestCase extends LuceneTestCase {* preserving the order of the added documents.*/private Directory getStableIndexStore(String field, float[]... contents) throws IOException {+ return getStableIndexStore(field, VectorSimilarityFunction.EUCLIDEAN, contents);+ }++ private Directory getStableIndexStore(+ String field, VectorSimilarityFunction similarityFunction, float[][] contents)+ throws IOException {Directory indexStore = newDirectoryForTest();- try (IndexWriter writer = new IndexWriter(indexStore, new IndexWriterConfig())) {+ try (IndexWriter writer = new IndexWriter(indexStore, configStandardCodec())) {for (int i = 0; i < contents.length; ++i) {Document doc = new Document();- doc.add(getKnnVectorField(field, contents[i]));+ doc.add(getKnnVectorField(field, contents[i], similarityFunction));doc.add(new StringField("id", "id" + i, Field.Store.YES));writer.addDocument(doc);}diff --git a/lucene/core/src/test/org/apache/lucene/search/BaseVectorSimilarityQueryTestCase.java b/lucene/core/src/test/org/apache/lucene/search/BaseVectorSimilarityQueryTestCase.javaindex c3623d9c28d..06b327e9afa 100644--- a/lucene/core/src/test/org/apache/lucene/search/BaseVectorSimilarityQueryTestCase.java+++ b/lucene/core/src/test/org/apache/lucene/search/BaseVectorSimilarityQueryTestCase.java@@ -34,6 +34,7 @@ import org.apache.lucene.document.IntField;import org.apache.lucene.index.DirectoryReader;import org.apache.lucene.index.IndexReader;import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.index.QueryTimeout;import org.apache.lucene.index.Term;@@ -41,6 +42,7 @@ import org.apache.lucene.index.VectorSimilarityFunction;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.index.RandomIndexWriter;import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.hnsw.HnswUtil;@LuceneTestCase.SuppressCodecs("SimpleText")@@ -381,7 +383,7 @@ abstract class BaseVectorSimilarityQueryTestCase<}}- public void testVectorsAboveSimilarity() throws IOException {+ void testVectorsAboveSimilarity() throws IOException {// Pick number of docs to acceptint numAccepted = random().nextInt(numDocs / 3, numDocs / 2);float delta = 1e-3f;@@ -401,7 +403,10 @@ abstract class BaseVectorSimilarityQueryTestCase<}}- try (Directory indexStore = getIndexStore(vectors);+ // TODO test with random codec params via getIndexStore(vectors);+ // this is challenging because scores will vary in a quantized index+ // and precomputing as above will not be accurate+ try (Directory indexStore = getStableIndexStore(vectors);IndexReader reader = DirectoryReader.open(indexStore)) {IndexSearcher searcher = newSearcher(reader);@@ -587,6 +592,21 @@ abstract class BaseVectorSimilarityQueryTestCase<return dir;}+ @SafeVarargs+ final Directory getStableIndexStore(V... vectors) throws IOException {+ Directory dir = newDirectory();+ IndexWriterConfig iwc = new IndexWriterConfig().setCodec(TestUtil.getDefaultCodec());+ try (IndexWriter writer = new IndexWriter(dir, iwc)) {+ for (int i = 0; i < vectors.length; ++i) {+ Document doc = new Document();+ doc.add(getVectorField(vectorField, vectors[i], function));+ doc.add(new IntField(idField, i, Field.Store.YES));+ writer.addDocument(doc);+ }+ }+ return dir;+ }+private static class CountingQueryTimeout implements QueryTimeout {private int remaining;diff --git a/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.javaindex feebe858c09..99572d2ecea 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.java@@ -44,6 +44,7 @@ import org.apache.lucene.util.TestVectorUtil;import org.apache.lucene.util.VectorUtil;public class TestKnnFloatVectorQuery extends BaseKnnVectorQueryTestCase {+@OverrideKnnFloatVectorQuery getKnnVectorQuery(String field, float[] query, int k, Query queryFilter) {return new KnnFloatVectorQuery(field, query, k, queryFilter);@@ -130,16 +131,16 @@ public class TestKnnFloatVectorQuery extends BaseKnnVectorQueryTestCase {DocIdSetIterator it = scorer.iterator();assertEquals(2, it.cost());assertEquals(0, it.nextDoc());- assertEquals(0, scorer.score(), 0);+ assertEquals(0, scorer.score(), EPSILON);assertEquals(1, it.advance(1));- assertEquals(1, scorer.score(), 0);+ assertEquals(1, scorer.score(), EPSILON);}}}public void testScoreDotProduct() throws IOException {try (Directory d = newDirectory()) {- try (IndexWriter w = new IndexWriter(d, new IndexWriterConfig())) {+ try (IndexWriter w = new IndexWriter(d, configStandardCodec())) {for (int j = 1; j <= 5; j++) {Document doc = new Document();doc.add(@@ -174,7 +175,7 @@ public class TestKnnFloatVectorQuery extends BaseKnnVectorQueryTestCase {(float) ((1 + (2 * 2 + 3 * 4) / Math.sqrt((2 * 2 + 3 * 3) * (2 * 2 + 4 * 4))) / 2);// doc 1 happens to have the max score- assertEquals(score1, scorer.getMaxScore(2), 0.0001);+ assertEquals(score1, scorer.getMaxScore(2), 0.001);assertEquals(score1, scorer.getMaxScore(Integer.MAX_VALUE), 0.0001);DocIdSetIterator it = scorer.iterator();diff --git a/lucene/core/src/test/org/apache/lucene/search/TestVectorSimilarityValuesSource.java b/lucene/core/src/test/org/apache/lucene/search/TestVectorSimilarityValuesSource.javaindex 11e6af63a52..eaf67279ee4 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestVectorSimilarityValuesSource.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestVectorSimilarityValuesSource.java@@ -18,6 +18,7 @@package org.apache.lucene.search;import org.apache.lucene.analysis.Analyzer;+import org.apache.lucene.codecs.KnnVectorsFormat;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.document.KnnByteVectorField;@@ -29,8 +30,10 @@ import org.apache.lucene.index.IndexWriterConfig;import org.apache.lucene.index.VectorSimilarityFunction;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.analysis.MockAnalyzer;+import org.apache.lucene.tests.codecs.asserting.AssertingCodec;import org.apache.lucene.tests.index.RandomIndexWriter;import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.IOUtils;import org.junit.AfterClass;@@ -47,6 +50,13 @@ public class TestVectorSimilarityValuesSource extends LuceneTestCase {dir = newDirectory();analyzer = new MockAnalyzer(random());IndexWriterConfig iwConfig = newIndexWriterConfig(analyzer);+ iwConfig.setCodec(+ new AssertingCodec() {+ @Override+ public KnnVectorsFormat getKnnVectorsFormatForField(String field) {+ return TestUtil.getDefaultKnnVectorsFormat();+ }+ });iwConfig.setMergePolicy(newLogMergePolicy());RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConfig);diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.javaindex 6e333c0cbd7..5ecca4746de 100644--- a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java@@ -27,6 +27,7 @@ import java.util.SortedMap;import java.util.TreeMap;import java.util.regex.Matcher;import java.util.regex.Pattern;+import org.apache.lucene.codecs.KnnVectorsFormat;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.document.Field.Store;@@ -56,6 +57,7 @@ import org.apache.lucene.search.similarities.Similarity;import org.apache.lucene.search.similarities.SimilarityBase;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.analysis.MockAnalyzer;+import org.apache.lucene.tests.codecs.asserting.AssertingCodec;import org.apache.lucene.tests.index.RandomIndexWriter;import org.apache.lucene.tests.search.CheckHits;import org.apache.lucene.tests.search.QueryUtils;@@ -252,7 +254,17 @@ public class TestBlockJoin extends LuceneTestCase {final Directory dir = newDirectory();final RandomIndexWriter w =new RandomIndexWriter(- random(), dir, newIndexWriterConfig().setMergePolicy(newMergePolicy(random(), false)));+ random(),+ dir,+ newIndexWriterConfig()+ .setCodec(+ new AssertingCodec() {+ @Override+ public KnnVectorsFormat getKnnVectorsFormatForField(String field) {+ return TestUtil.getDefaultKnnVectorsFormat();+ }+ })+ .setMergePolicy(newMergePolicy(random(), false)));final List<Document> docs = new ArrayList<>();diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.javaindex fbc54c30954..91bfa819979 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.java@@ -25,7 +25,9 @@ import java.util.List;import java.util.Map;import java.util.Random;import java.util.Set;+import java.util.concurrent.ForkJoinPool;import org.apache.lucene.codecs.DocValuesFormat;+import org.apache.lucene.codecs.KnnVectorsFormat;import org.apache.lucene.codecs.PointsFormat;import org.apache.lucene.codecs.PointsReader;import org.apache.lucene.codecs.PointsWriter;@@ -34,6 +36,8 @@ import org.apache.lucene.codecs.blocktreeords.BlockTreeOrdsPostingsFormat;import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;import org.apache.lucene.codecs.lucene90.Lucene90PointsReader;import org.apache.lucene.codecs.lucene90.Lucene90PointsWriter;+import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;import org.apache.lucene.codecs.memory.DirectPostingsFormat;import org.apache.lucene.codecs.memory.FSTPostingsFormat;import org.apache.lucene.index.FieldInfo;@@ -44,6 +48,7 @@ import org.apache.lucene.index.SegmentWriteState;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.codecs.asserting.AssertingCodec;import org.apache.lucene.tests.codecs.asserting.AssertingDocValuesFormat;+import org.apache.lucene.tests.codecs.asserting.AssertingKnnVectorsFormat;import org.apache.lucene.tests.codecs.asserting.AssertingPointsFormat;import org.apache.lucene.tests.codecs.asserting.AssertingPostingsFormat;import org.apache.lucene.tests.codecs.blockterms.LuceneFixedGap;@@ -71,12 +76,18 @@ public class RandomCodec extends AssertingCodec {/** Shuffled list of docvalues formats to use for new mappings */private List<DocValuesFormat> dvFormats = new ArrayList<>();+ /** Shuffled list of knn formats to use for new mappings */+ private List<KnnVectorsFormat> knnFormats = new ArrayList<>();+/** unique set of format names this codec knows about */public Set<String> formatNames = new HashSet<>();/** unique set of docvalues format names this codec knows about */public Set<String> dvFormatNames = new HashSet<>();+ /** unique set of knn format names this codec knows about */+ public Set<String> knnFormatNames = new HashSet<>();+public final Set<String> avoidCodecs;/** memorized field to postingsformat mappings */@@ -88,6 +99,10 @@ public class RandomCodec extends AssertingCodec {private Map<String, DocValuesFormat> previousDVMappings =Collections.synchronizedMap(new HashMap<String, DocValuesFormat>());++ private Map<String, KnnVectorsFormat> previousKnnMappings =+ Collections.synchronizedMap(new HashMap<String, KnnVectorsFormat>());+private final int perFieldSeed;// a little messy: randomize the default codec's parameters here.@@ -191,6 +206,18 @@ public class RandomCodec extends AssertingCodec {return codec;}+ @Override+ public KnnVectorsFormat getKnnVectorsFormatForField(String name) {+ KnnVectorsFormat format = previousKnnMappings.get(name);+ if (format == null) {+ format = knnFormats.get(Math.abs(perFieldSeed ^ name.hashCode()) % knnFormats.size());+ previousKnnMappings.put(name, format);+ // Safety:+ assert previousKnnMappings.size() < 10000 : "test went insane";+ }+ return format;+ }+public RandomCodec(Random random, Set<String> avoidCodecs) {this.perFieldSeed = random.nextInt();this.avoidCodecs = avoidCodecs;@@ -236,8 +263,46 @@ public class RandomCodec extends AssertingCodec {new Lucene90DocValuesFormat(),new AssertingDocValuesFormat());+ boolean concurrentKnnMerging = random.nextBoolean();+ addKnn(+ avoidCodecs,+ TestUtil.getDefaultKnnVectorsFormat(),+ new Lucene99HnswVectorsFormat(+ TestUtil.nextInt(random, 5, 50),+ TestUtil.nextInt(random, 10, 50),+ concurrentKnnMerging ? TestUtil.nextInt(random, 2, 8) : 1,+ concurrentKnnMerging ? ForkJoinPool.commonPool() : null),+ new Lucene99HnswScalarQuantizedVectorsFormat(+ TestUtil.nextInt(random, 5, 50),+ TestUtil.nextInt(random, 10, 50),+ concurrentKnnMerging ? TestUtil.nextInt(random, 2, 8) : 1,+ 7,+ false,+ randomConfidenceInterval(random),+ concurrentKnnMerging ? ForkJoinPool.commonPool() : null),+ // TODO: also test 4-bit quantization, but this must somehow be restricted to even-length+ // fields+ /*+ new Lucene99HnswScalarQuantizedVectorsFormat(TestUtil.nextInt(random, 5, 50),+ TestUtil.nextInt(random, 10, 50),+ 1,+ 4,+ random.nextBoolean(),+ randomConfidenceInterval(random),+ null),+ new Lucene99HnswScalarQuantizedVectorsFormat(TestUtil.nextInt(random, 5, 50),+ TestUtil.nextInt(random, 10, 50),+ TestUtil.nextInt(random, 2, 8),+ 4,+ random.nextBoolean(),+ randomConfidenceInterval(random),+ ForkJoinPool.commonPool()),+ */+ new AssertingKnnVectorsFormat());+Collections.shuffle(formats, random);Collections.shuffle(dvFormats, random);+ Collections.shuffle(knnFormats, random);// Avoid too many open files:if (formats.size() > 4) {@@ -246,6 +311,21 @@ public class RandomCodec extends AssertingCodec {if (dvFormats.size() > 4) {dvFormats = dvFormats.subList(0, 4);}+ if (knnFormats.size() > 4) {+ knnFormats = knnFormats.subList(0, 4);+ }+ }++ private final Float randomConfidenceInterval(Random random) {+ switch (random.nextInt(3)) {+ default:+ case 0:+ return null;+ case 1:+ return 0f;+ case 2:+ return random.nextFloat(0.9f, 1f);+ }}public RandomCodec(Random random) {@@ -270,6 +350,15 @@ public class RandomCodec extends AssertingCodec {}}+ private final void addKnn(Set<String> avoidCodecs, KnnVectorsFormat... knnFormat) {+ for (KnnVectorsFormat kf : knnFormat) {+ if (!avoidCodecs.contains(kf.getName())) {+ knnFormats.add(kf);+ knnFormatNames.add(kf.getName());+ }+ }+ }+@Overridepublic String toString() {return super.toString()