Skip to content

Apache Lucene open-source search software

Randomize KnnVector codec params in RandomCodec; addresses gh-14047 (#14049)

Michael Sokolov6b0112cdee28
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 5bb515d3cba..a76302c5742 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -24,7 +24,8 @@ Optimizations
Bug Fixes
---------------------
-(No changes)
+* GITHUB#14049: Randomize KNN codec params in RandomCodec. Fixes scalar quantization div-by-zero
+ when all values are identical. (Mike Sokolov)
Other
---------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBasicBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBasicBackwardsCompatibility.java
index 262567f9f76..a93c5c7617a 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBasicBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBasicBackwardsCompatibility.java
@@ -541,6 +541,7 @@ public class TestBasicBackwardsCompatibility extends BackwardsCompatibilityTestB
new IndexWriter(
dir,
newIndexWriterConfig(new MockAnalyzer(random))
+ .setCodec(TestUtil.getDefaultCodec())
.setOpenMode(IndexWriterConfig.OpenMode.APPEND)
.setMergePolicy(newLogMergePolicy()));
// add 10 docs
@@ -579,6 +580,7 @@ public class TestBasicBackwardsCompatibility extends BackwardsCompatibilityTestB
new IndexWriter(
dir,
newIndexWriterConfig(new MockAnalyzer(random))
+ .setCodec(TestUtil.getDefaultCodec())
.setOpenMode(IndexWriterConfig.OpenMode.APPEND)
.setMergePolicy(newLogMergePolicy()));
writer.forceMerge(1);
@@ -630,6 +632,7 @@ public class TestBasicBackwardsCompatibility extends BackwardsCompatibilityTestB
new IndexWriter(
dir,
newIndexWriterConfig(new MockAnalyzer(random))
+ .setCodec(TestUtil.getDefaultCodec())
.setOpenMode(IndexWriterConfig.OpenMode.APPEND));
writer.forceMerge(1);
writer.close();
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java
index a4770f01f46..8fc417e22f0 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorScorer.java
@@ -93,6 +93,7 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {
VectorSimilarityFunction sim,
float constMultiplier,
QuantizedByteVectorValues values) {
+ checkDimensions(targetBytes.length, values.dimension());
return switch (sim) {
case EUCLIDEAN -> new Euclidean(values, constMultiplier, targetBytes);
case COSINE, DOT_PRODUCT ->
@@ -112,6 +113,13 @@ public class Lucene99ScalarQuantizedVectorScorer implements FlatVectorsScorer {
};
}
+ static void checkDimensions(int queryLen, int fieldLen) {
+ if (queryLen != fieldLen) {
+ throw new IllegalArgumentException(
+ "vector query dimension: " + queryLen + " differs from field dimension: " + fieldLen);
+ }
+ }
+
private static RandomVectorScorer.AbstractRandomVectorScorer dotProductFactory(
byte[] targetBytes,
float offsetCorrection,
diff --git a/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java b/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java
index 3f7bcf6c5c4..34fb380a183 100644
--- a/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java
+++ b/lucene/core/src/java/org/apache/lucene/util/quantization/ScalarQuantizer.java
@@ -97,12 +97,18 @@ public class ScalarQuantizer {
}
assert maxQuantile >= minQuantile;
assert bits > 0 && bits <= 8;
- this.minQuantile = minQuantile;
- this.maxQuantile = maxQuantile;
this.bits = bits;
final float divisor = (float) ((1 << bits) - 1);
- this.scale = divisor / (maxQuantile - minQuantile);
- this.alpha = (maxQuantile - minQuantile) / divisor;
+ if (minQuantile == maxQuantile) {
+ // avoid divide-by-zero with an arbitrary but plausible choice (leads to alpha = scale = 1)
+ this.minQuantile = minQuantile - divisor;
+ this.maxQuantile = maxQuantile + divisor;
+ } else {
+ this.minQuantile = minQuantile;
+ this.maxQuantile = maxQuantile;
+ }
+ this.scale = divisor / (this.maxQuantile - this.minQuantile);
+ this.alpha = (this.maxQuantile - this.minQuantile) / divisor;
}
/**
diff --git a/lucene/core/src/test/org/apache/lucene/search/BaseKnnVectorQueryTestCase.java b/lucene/core/src/test/org/apache/lucene/search/BaseKnnVectorQueryTestCase.java
index 4abd955eeb6..1700a1dbae8 100644
--- a/lucene/core/src/test/org/apache/lucene/search/BaseKnnVectorQueryTestCase.java
+++ b/lucene/core/src/test/org/apache/lucene/search/BaseKnnVectorQueryTestCase.java
@@ -19,7 +19,6 @@ package org.apache.lucene.search;
import static com.carrotsearch.randomizedtesting.RandomizedTest.frequently;
import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean;
import static com.carrotsearch.randomizedtesting.RandomizedTest.randomIntBetween;
-import static org.apache.lucene.index.VectorSimilarityFunction.COSINE;
import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
import java.io.IOException;
@@ -61,6 +60,8 @@ import org.apache.lucene.util.FixedBitSet;
/** Test cases for AbstractKnnVectorQuery objects. */
abstract class BaseKnnVectorQueryTestCase extends LuceneTestCase {
+ // handle quantization noise
+ static final float EPSILON = 0.001f;
abstract AbstractKnnVectorQuery getKnnVectorQuery(
String field, float[] query, int k, Query queryFilter);
@@ -86,6 +87,10 @@ abstract class BaseKnnVectorQueryTestCase extends LuceneTestCase {
return LuceneTestCase.newDirectory(random());
}
+ protected IndexWriterConfig configStandardCodec() throws IOException {
+ return new IndexWriterConfig().setCodec(TestUtil.getDefaultCodec());
+ }
+
public void testEquals() {
AbstractKnnVectorQuery q1 = getKnnVectorQuery("f1", new float[] {0, 1}, 10);
Query filter1 = new TermQuery(new Term("id", "id1"));
@@ -317,66 +322,58 @@ abstract class BaseKnnVectorQueryTestCase extends LuceneTestCase {
}
public void testScoreCosine() throws IOException {
- try (Directory d = newDirectoryForTest()) {
- try (IndexWriter w = new IndexWriter(d, new IndexWriterConfig())) {
- for (int j = 1; j <= 5; j++) {
- Document doc = new Document();
- doc.add(getKnnVectorField("field", new float[] {j, j * j}, COSINE));
- w.addDocument(doc);
- }
- }
- try (IndexReader reader = DirectoryReader.open(d)) {
- assertEquals(1, reader.leaves().size());
- IndexSearcher searcher = new IndexSearcher(reader);
- AbstractKnnVectorQuery query = getKnnVectorQuery("field", new float[] {2, 3}, 3);
- Query rewritten = query.rewrite(searcher);
- Weight weight = searcher.createWeight(rewritten, ScoreMode.COMPLETE, 1);
- Scorer scorer = weight.scorer(reader.leaves().get(0));
-
- // prior to advancing, score is undefined
- assertEquals(-1, scorer.docID());
- expectThrows(ArrayIndexOutOfBoundsException.class, scorer::score);
-
- /* score0 = ((2,3) * (1, 1) = 5) / (||2, 3|| * ||1, 1|| = sqrt(26)), then
- * normalized by (1 + x) /2.
- */
- float score0 =
- (float) ((1 + (2 * 1 + 3 * 1) / Math.sqrt((2 * 2 + 3 * 3) * (1 * 1 + 1 * 1))) / 2);
-
- /* score1 = ((2,3) * (2, 4) = 16) / (||2, 3|| * ||2, 4|| = sqrt(260)), then
- * normalized by (1 + x) /2
- */
- float score1 =
- (float) ((1 + (2 * 2 + 3 * 4) / Math.sqrt((2 * 2 + 3 * 3) * (2 * 2 + 4 * 4))) / 2);
-
- // doc 1 happens to have the maximum score
- assertEquals(score1, scorer.getMaxScore(2), 0.0001);
- assertEquals(score1, scorer.getMaxScore(Integer.MAX_VALUE), 0.0001);
-
- DocIdSetIterator it = scorer.iterator();
- assertEquals(3, it.cost());
- assertEquals(0, it.nextDoc());
- // doc 0 has (1, 1)
- assertEquals(score0, scorer.score(), 0.0001);
- assertEquals(1, it.advance(1));
- assertEquals(score1, scorer.score(), 0.0001);
-
- // since topK was 3
- assertEquals(NO_MORE_DOCS, it.advance(4));
- expectThrows(ArrayIndexOutOfBoundsException.class, scorer::score);
- }
+ float[][] vectors = new float[5][];
+ for (int j = 1; j <= 5; j++) {
+ vectors[j - 1] = new float[] {j, j * j};
+ }
+ try (Directory d = getStableIndexStore("field", VectorSimilarityFunction.COSINE, vectors);
+ IndexReader reader = DirectoryReader.open(d)) {
+ assertEquals(1, reader.leaves().size());
+ IndexSearcher searcher = new IndexSearcher(reader);
+ AbstractKnnVectorQuery query = getKnnVectorQuery("field", new float[] {2, 3}, 3);
+ Query rewritten = query.rewrite(searcher);
+ Weight weight = searcher.createWeight(rewritten, ScoreMode.COMPLETE, 1);
+ Scorer scorer = weight.scorer(reader.leaves().get(0));
+
+ // prior to advancing, score is undefined
+ assertEquals(-1, scorer.docID());
+ expectThrows(ArrayIndexOutOfBoundsException.class, scorer::score);
+
+ /* score0 = ((2,3) * (1, 1) = 5) / (||2, 3|| * ||1, 1|| = sqrt(26)), then
+ * normalized by (1 + x) /2.
+ */
+ float score0 =
+ (float) ((1 + (2 * 1 + 3 * 1) / Math.sqrt((2 * 2 + 3 * 3) * (1 * 1 + 1 * 1))) / 2);
+
+ /* score1 = ((2,3) * (2, 4) = 16) / (||2, 3|| * ||2, 4|| = sqrt(260)), then
+ * normalized by (1 + x) /2
+ */
+ float score1 =
+ (float) ((1 + (2 * 2 + 3 * 4) / Math.sqrt((2 * 2 + 3 * 3) * (2 * 2 + 4 * 4))) / 2);
+
+ // doc 1 happens to have the maximum score
+ assertEquals(score1, scorer.getMaxScore(2), 0.0001);
+ assertEquals(score1, scorer.getMaxScore(Integer.MAX_VALUE), 0.0001);
+
+ DocIdSetIterator it = scorer.iterator();
+ assertEquals(3, it.cost());
+ assertEquals(0, it.nextDoc());
+ // doc 0 has (1, 1)
+ assertEquals(score0, scorer.score(), 0.0001);
+ assertEquals(1, it.advance(1));
+ assertEquals(score1, scorer.score(), 0.0001);
+
+ // since topK was 3
+ assertEquals(NO_MORE_DOCS, it.advance(4));
+ expectThrows(ArrayIndexOutOfBoundsException.class, scorer::score);
}
}
public void testScoreMIP() throws IOException {
- try (Directory indexStore =
- getIndexStore(
- "field",
- VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT,
- new float[] {0, 1},
- new float[] {1, 2},
- new float[] {0, 0});
- IndexReader reader = DirectoryReader.open(indexStore)) {
+ float[][] vectors = {{0, 1}, {1, 2}, {0, 0}};
+ try (Directory d =
+ getStableIndexStore("field", VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT, vectors);
+ IndexReader reader = DirectoryReader.open(d)) {
IndexSearcher searcher = newSearcher(reader);
AbstractKnnVectorQuery kvq = getKnnVectorQuery("field", new float[] {0, -1}, 10);
assertMatches(searcher, kvq, 3);
@@ -405,7 +402,8 @@ abstract class BaseKnnVectorQueryTestCase extends LuceneTestCase {
AbstractKnnVectorQuery query = getKnnVectorQuery("field", new float[] {2, 3}, 3);
Explanation matched = searcher.explain(query, 2);
assertTrue(matched.isMatch());
- assertEquals(1 / 2f, matched.getValue());
+ // scores vary widely due to quantization
+ assertEquals(1 / 2f, matched.getValue().doubleValue(), 0.5);
assertEquals(0, matched.getDetails().length);
assertEquals("within top 3 docs", matched.getDescription());
@@ -431,9 +429,10 @@ abstract class BaseKnnVectorQueryTestCase extends LuceneTestCase {
try (IndexReader reader = DirectoryReader.open(d)) {
IndexSearcher searcher = new IndexSearcher(reader);
AbstractKnnVectorQuery query = getKnnVectorQuery("field", new float[] {2, 3}, 3);
- Explanation matched = searcher.explain(query, 2);
+ Explanation matched = searcher.explain(query, 2); // (2, 2)
assertTrue(matched.isMatch());
- assertEquals(1 / 2f, matched.getValue());
+ // scores vary widely due to quantization
+ assertEquals(1 / 2f, matched.getValue().doubleValue(), 0.5);
assertEquals(0, matched.getDetails().length);
assertEquals("within top 3 docs", matched.getDescription());
@@ -453,7 +452,7 @@ abstract class BaseKnnVectorQueryTestCase extends LuceneTestCase {
* randomly fail to find one).
*/
try (Directory d = newDirectoryForTest()) {
- try (IndexWriter w = new IndexWriter(d, new IndexWriterConfig())) {
+ try (IndexWriter w = new IndexWriter(d, configStandardCodec())) {
int r = 0;
for (int i = 0; i < 5; i++) {
for (int j = 0; j < 5; j++) {
@@ -532,7 +531,7 @@ abstract class BaseKnnVectorQueryTestCase extends LuceneTestCase {
// visitedLimit. This is fine since the test targets AbstractKnnVectorQuery logic, not the kNN
// format
// implementation.
- IndexWriterConfig iwc = new IndexWriterConfig().setCodec(TestUtil.getDefaultCodec());
+ IndexWriterConfig iwc = configStandardCodec();
RandomIndexWriter w = new RandomIndexWriter(random(), d, iwc);
for (int i = 0; i < numDocs; i++) {
Document doc = new Document();
@@ -618,7 +617,7 @@ abstract class BaseKnnVectorQueryTestCase extends LuceneTestCase {
// visitedLimit. This is fine since the test targets AbstractKnnVectorQuery logic, not the kNN
// format
// implementation.
- IndexWriterConfig iwc = new IndexWriterConfig().setCodec(TestUtil.getDefaultCodec());
+ IndexWriterConfig iwc = configStandardCodec();
IndexWriter w = new IndexWriter(d, iwc);
float[] vector = randomVector(dimension);
for (int i = 0; i < numDocs; i++) {
@@ -933,11 +932,17 @@ abstract class BaseKnnVectorQueryTestCase extends LuceneTestCase {
* preserving the order of the added documents.
*/
private Directory getStableIndexStore(String field, float[]... contents) throws IOException {
+ return getStableIndexStore(field, VectorSimilarityFunction.EUCLIDEAN, contents);
+ }
+
+ private Directory getStableIndexStore(
+ String field, VectorSimilarityFunction similarityFunction, float[][] contents)
+ throws IOException {
Directory indexStore = newDirectoryForTest();
- try (IndexWriter writer = new IndexWriter(indexStore, new IndexWriterConfig())) {
+ try (IndexWriter writer = new IndexWriter(indexStore, configStandardCodec())) {
for (int i = 0; i < contents.length; ++i) {
Document doc = new Document();
- doc.add(getKnnVectorField(field, contents[i]));
+ doc.add(getKnnVectorField(field, contents[i], similarityFunction));
doc.add(new StringField("id", "id" + i, Field.Store.YES));
writer.addDocument(doc);
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/BaseVectorSimilarityQueryTestCase.java b/lucene/core/src/test/org/apache/lucene/search/BaseVectorSimilarityQueryTestCase.java
index c3623d9c28d..06b327e9afa 100644
--- a/lucene/core/src/test/org/apache/lucene/search/BaseVectorSimilarityQueryTestCase.java
+++ b/lucene/core/src/test/org/apache/lucene/search/BaseVectorSimilarityQueryTestCase.java
@@ -34,6 +34,7 @@ import org.apache.lucene.document.IntField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.QueryTimeout;
import org.apache.lucene.index.Term;
@@ -41,6 +42,7 @@ import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.hnsw.HnswUtil;
@LuceneTestCase.SuppressCodecs("SimpleText")
@@ -381,7 +383,7 @@ abstract class BaseVectorSimilarityQueryTestCase<
}
}
- public void testVectorsAboveSimilarity() throws IOException {
+ void testVectorsAboveSimilarity() throws IOException {
// Pick number of docs to accept
int numAccepted = random().nextInt(numDocs / 3, numDocs / 2);
float delta = 1e-3f;
@@ -401,7 +403,10 @@ abstract class BaseVectorSimilarityQueryTestCase<
}
}
- try (Directory indexStore = getIndexStore(vectors);
+ // TODO test with random codec params via getIndexStore(vectors);
+ // this is challenging because scores will vary in a quantized index
+ // and precomputing as above will not be accurate
+ try (Directory indexStore = getStableIndexStore(vectors);
IndexReader reader = DirectoryReader.open(indexStore)) {
IndexSearcher searcher = newSearcher(reader);
@@ -587,6 +592,21 @@ abstract class BaseVectorSimilarityQueryTestCase<
return dir;
}
+ @SafeVarargs
+ final Directory getStableIndexStore(V... vectors) throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig().setCodec(TestUtil.getDefaultCodec());
+ try (IndexWriter writer = new IndexWriter(dir, iwc)) {
+ for (int i = 0; i < vectors.length; ++i) {
+ Document doc = new Document();
+ doc.add(getVectorField(vectorField, vectors[i], function));
+ doc.add(new IntField(idField, i, Field.Store.YES));
+ writer.addDocument(doc);
+ }
+ }
+ return dir;
+ }
+
private static class CountingQueryTimeout implements QueryTimeout {
private int remaining;
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.java
index feebe858c09..99572d2ecea 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.java
@@ -44,6 +44,7 @@ import org.apache.lucene.util.TestVectorUtil;
import org.apache.lucene.util.VectorUtil;
public class TestKnnFloatVectorQuery extends BaseKnnVectorQueryTestCase {
+
@Override
KnnFloatVectorQuery getKnnVectorQuery(String field, float[] query, int k, Query queryFilter) {
return new KnnFloatVectorQuery(field, query, k, queryFilter);
@@ -130,16 +131,16 @@ public class TestKnnFloatVectorQuery extends BaseKnnVectorQueryTestCase {
DocIdSetIterator it = scorer.iterator();
assertEquals(2, it.cost());
assertEquals(0, it.nextDoc());
- assertEquals(0, scorer.score(), 0);
+ assertEquals(0, scorer.score(), EPSILON);
assertEquals(1, it.advance(1));
- assertEquals(1, scorer.score(), 0);
+ assertEquals(1, scorer.score(), EPSILON);
}
}
}
public void testScoreDotProduct() throws IOException {
try (Directory d = newDirectory()) {
- try (IndexWriter w = new IndexWriter(d, new IndexWriterConfig())) {
+ try (IndexWriter w = new IndexWriter(d, configStandardCodec())) {
for (int j = 1; j <= 5; j++) {
Document doc = new Document();
doc.add(
@@ -174,7 +175,7 @@ public class TestKnnFloatVectorQuery extends BaseKnnVectorQueryTestCase {
(float) ((1 + (2 * 2 + 3 * 4) / Math.sqrt((2 * 2 + 3 * 3) * (2 * 2 + 4 * 4))) / 2);
// doc 1 happens to have the max score
- assertEquals(score1, scorer.getMaxScore(2), 0.0001);
+ assertEquals(score1, scorer.getMaxScore(2), 0.001);
assertEquals(score1, scorer.getMaxScore(Integer.MAX_VALUE), 0.0001);
DocIdSetIterator it = scorer.iterator();
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestVectorSimilarityValuesSource.java b/lucene/core/src/test/org/apache/lucene/search/TestVectorSimilarityValuesSource.java
index 11e6af63a52..eaf67279ee4 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestVectorSimilarityValuesSource.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestVectorSimilarityValuesSource.java
@@ -18,6 +18,7 @@
package org.apache.lucene.search;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.KnnByteVectorField;
@@ -29,8 +30,10 @@ import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
+import org.apache.lucene.tests.codecs.asserting.AssertingCodec;
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.junit.AfterClass;
@@ -47,6 +50,13 @@ public class TestVectorSimilarityValuesSource extends LuceneTestCase {
dir = newDirectory();
analyzer = new MockAnalyzer(random());
IndexWriterConfig iwConfig = newIndexWriterConfig(analyzer);
+ iwConfig.setCodec(
+ new AssertingCodec() {
+ @Override
+ public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
+ return TestUtil.getDefaultKnnVectorsFormat();
+ }
+ });
iwConfig.setMergePolicy(newLogMergePolicy());
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConfig);
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java
index 6e333c0cbd7..5ecca4746de 100644
--- a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java
+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java
@@ -27,6 +27,7 @@ import java.util.SortedMap;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
@@ -56,6 +57,7 @@ import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.SimilarityBase;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
+import org.apache.lucene.tests.codecs.asserting.AssertingCodec;
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.tests.search.CheckHits;
import org.apache.lucene.tests.search.QueryUtils;
@@ -252,7 +254,17 @@ public class TestBlockJoin extends LuceneTestCase {
final Directory dir = newDirectory();
final RandomIndexWriter w =
new RandomIndexWriter(
- random(), dir, newIndexWriterConfig().setMergePolicy(newMergePolicy(random(), false)));
+ random(),
+ dir,
+ newIndexWriterConfig()
+ .setCodec(
+ new AssertingCodec() {
+ @Override
+ public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
+ return TestUtil.getDefaultKnnVectorsFormat();
+ }
+ })
+ .setMergePolicy(newMergePolicy(random(), false)));
final List<Document> docs = new ArrayList<>();
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.java
index fbc54c30954..91bfa819979 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomCodec.java
@@ -25,7 +25,9 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
+import java.util.concurrent.ForkJoinPool;
import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.PointsFormat;
import org.apache.lucene.codecs.PointsReader;
import org.apache.lucene.codecs.PointsWriter;
@@ -34,6 +36,8 @@ import org.apache.lucene.codecs.blocktreeords.BlockTreeOrdsPostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.apache.lucene.codecs.lucene90.Lucene90PointsReader;
import org.apache.lucene.codecs.lucene90.Lucene90PointsWriter;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
import org.apache.lucene.codecs.memory.DirectPostingsFormat;
import org.apache.lucene.codecs.memory.FSTPostingsFormat;
import org.apache.lucene.index.FieldInfo;
@@ -44,6 +48,7 @@ import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.codecs.asserting.AssertingCodec;
import org.apache.lucene.tests.codecs.asserting.AssertingDocValuesFormat;
+import org.apache.lucene.tests.codecs.asserting.AssertingKnnVectorsFormat;
import org.apache.lucene.tests.codecs.asserting.AssertingPointsFormat;
import org.apache.lucene.tests.codecs.asserting.AssertingPostingsFormat;
import org.apache.lucene.tests.codecs.blockterms.LuceneFixedGap;
@@ -71,12 +76,18 @@ public class RandomCodec extends AssertingCodec {
/** Shuffled list of docvalues formats to use for new mappings */
private List<DocValuesFormat> dvFormats = new ArrayList<>();
+ /** Shuffled list of knn formats to use for new mappings */
+ private List<KnnVectorsFormat> knnFormats = new ArrayList<>();
+
/** unique set of format names this codec knows about */
public Set<String> formatNames = new HashSet<>();
/** unique set of docvalues format names this codec knows about */
public Set<String> dvFormatNames = new HashSet<>();
+ /** unique set of knn format names this codec knows about */
+ public Set<String> knnFormatNames = new HashSet<>();
+
public final Set<String> avoidCodecs;
/** memorized field to postingsformat mappings */
@@ -88,6 +99,10 @@ public class RandomCodec extends AssertingCodec {
private Map<String, DocValuesFormat> previousDVMappings =
Collections.synchronizedMap(new HashMap<String, DocValuesFormat>());
+
+ private Map<String, KnnVectorsFormat> previousKnnMappings =
+ Collections.synchronizedMap(new HashMap<String, KnnVectorsFormat>());
+
private final int perFieldSeed;
// a little messy: randomize the default codec's parameters here.
@@ -191,6 +206,18 @@ public class RandomCodec extends AssertingCodec {
return codec;
}
+ @Override
+ public KnnVectorsFormat getKnnVectorsFormatForField(String name) {
+ KnnVectorsFormat format = previousKnnMappings.get(name);
+ if (format == null) {
+ format = knnFormats.get(Math.abs(perFieldSeed ^ name.hashCode()) % knnFormats.size());
+ previousKnnMappings.put(name, format);
+ // Safety:
+ assert previousKnnMappings.size() < 10000 : "test went insane";
+ }
+ return format;
+ }
+
public RandomCodec(Random random, Set<String> avoidCodecs) {
this.perFieldSeed = random.nextInt();
this.avoidCodecs = avoidCodecs;
@@ -236,8 +263,46 @@ public class RandomCodec extends AssertingCodec {
new Lucene90DocValuesFormat(),
new AssertingDocValuesFormat());
+ boolean concurrentKnnMerging = random.nextBoolean();
+ addKnn(
+ avoidCodecs,
+ TestUtil.getDefaultKnnVectorsFormat(),
+ new Lucene99HnswVectorsFormat(
+ TestUtil.nextInt(random, 5, 50),
+ TestUtil.nextInt(random, 10, 50),
+ concurrentKnnMerging ? TestUtil.nextInt(random, 2, 8) : 1,
+ concurrentKnnMerging ? ForkJoinPool.commonPool() : null),
+ new Lucene99HnswScalarQuantizedVectorsFormat(
+ TestUtil.nextInt(random, 5, 50),
+ TestUtil.nextInt(random, 10, 50),
+ concurrentKnnMerging ? TestUtil.nextInt(random, 2, 8) : 1,
+ 7,
+ false,
+ randomConfidenceInterval(random),
+ concurrentKnnMerging ? ForkJoinPool.commonPool() : null),
+ // TODO: also test 4-bit quantization, but this must somehow be restricted to even-length
+ // fields
+ /*
+ new Lucene99HnswScalarQuantizedVectorsFormat(TestUtil.nextInt(random, 5, 50),
+ TestUtil.nextInt(random, 10, 50),
+ 1,
+ 4,
+ random.nextBoolean(),
+ randomConfidenceInterval(random),
+ null),
+ new Lucene99HnswScalarQuantizedVectorsFormat(TestUtil.nextInt(random, 5, 50),
+ TestUtil.nextInt(random, 10, 50),
+ TestUtil.nextInt(random, 2, 8),
+ 4,
+ random.nextBoolean(),
+ randomConfidenceInterval(random),
+ ForkJoinPool.commonPool()),
+ */
+ new AssertingKnnVectorsFormat());
+
Collections.shuffle(formats, random);
Collections.shuffle(dvFormats, random);
+ Collections.shuffle(knnFormats, random);
// Avoid too many open files:
if (formats.size() > 4) {
@@ -246,6 +311,21 @@ public class RandomCodec extends AssertingCodec {
if (dvFormats.size() > 4) {
dvFormats = dvFormats.subList(0, 4);
}
+ if (knnFormats.size() > 4) {
+ knnFormats = knnFormats.subList(0, 4);
+ }
+ }
+
+ private final Float randomConfidenceInterval(Random random) {
+ switch (random.nextInt(3)) {
+ default:
+ case 0:
+ return null;
+ case 1:
+ return 0f;
+ case 2:
+ return random.nextFloat(0.9f, 1f);
+ }
}
public RandomCodec(Random random) {
@@ -270,6 +350,15 @@ public class RandomCodec extends AssertingCodec {
}
}
+ private final void addKnn(Set<String> avoidCodecs, KnnVectorsFormat... knnFormat) {
+ for (KnnVectorsFormat kf : knnFormat) {
+ if (!avoidCodecs.contains(kf.getName())) {
+ knnFormats.add(kf);
+ knnFormatNames.add(kf.getName());
+ }
+ }
+ }
+
@Override
public String toString() {
return super.toString()