Skip to content

Apache Lucene open-source search software

Add ParentBlockJoinKnnVectorQueryTestCase.testRandom

Michael Sokolov76affdc78fcc
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/ParentBlockJoinKnnVectorQueryTestCase.java b/lucene/join/src/test/org/apache/lucene/search/join/ParentBlockJoinKnnVectorQueryTestCase.java
index 2594c20ab8e..023a5cd3ea2 100644
--- a/lucene/join/src/test/org/apache/lucene/search/join/ParentBlockJoinKnnVectorQueryTestCase.java
+++ b/lucene/join/src/test/org/apache/lucene/search/join/ParentBlockJoinKnnVectorQueryTestCase.java
@@ -28,6 +28,7 @@ import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
@@ -73,6 +74,8 @@ abstract class ParentBlockJoinKnnVectorQueryTestCase extends LuceneTestCase {
return parent;
}
+ abstract float[] randomVector(int dim);
+
abstract Query getParentJoinKnnQuery(
String fieldName, float[] queryVector, Query childFilter, int k, BitSetProducer parentBitSet);
@@ -108,8 +111,7 @@ abstract class ParentBlockJoinKnnVectorQueryTestCase extends LuceneTestCase {
try (IndexReader reader = DirectoryReader.open(d)) {
IndexSearcher searcher = new IndexSearcher(reader);
// Create parent filter directly, tests use "check" to verify parentIds exist. Production
- // may not
- // verify we handle it gracefully
+ // may not verify we handle it gracefully
BitSetProducer parentFilter =
new QueryBitSetProducer(new TermQuery(new Term("docType", "_parent")));
Query query = getParentJoinKnnQuery("field", new float[] {2, 2}, null, 3, parentFilter);
@@ -405,4 +407,73 @@ abstract class ParentBlockJoinKnnVectorQueryTestCase extends LuceneTestCase {
return true;
}
}
+
+ /** Tests with random vectors, number of documents, etc. Uses RandomIndexWriter. */
+ public void testRandom() throws IOException {
+ int numDocs = atLeast(100);
+ int dimension = atLeast(5);
+ int numIters = atLeast(10);
+ boolean everyDocHasAVector = random().nextBoolean();
+ int numParentsWithChildren = 0;
+ try (Directory d = newDirectory()) {
+ RandomIndexWriter w = new RandomIndexWriter(random(), d);
+ for (int i = 0; i < numDocs; i++) {
+ List<Document> family = new ArrayList<>();
+ Document doc = new Document();
+ if (random().nextInt(5) == 1) {
+ if (family.isEmpty() == false) {
+ ++numParentsWithChildren;
+ // System.out.println("parent w/children id=" + i);
+ doc.add(new StoredField("id", Integer.toString(i)));
+ } else {
+ doc.add(new StoredField("id", "pnoc" + Integer.toString(i)));
+ }
+ doc.add(new StringField("docType", "_parent", Field.Store.NO));
+ family.add(doc);
+ w.addDocuments(family);
+ family.clear();
+ } else if (everyDocHasAVector || random().nextInt(10) != 2) {
+ // NOTE: only child documents are allowed to have a vector!
+ // Otherwise the query's assumptions are invalidated??
+ doc.add(getKnnVectorField("field", randomVector(dimension)));
+ doc.add(new StoredField("id", "c" + Integer.toString(i)));
+ family.add(doc);
+ }
+ }
+ w.close();
+ // trailing children with no parent document are dropped
+ try (IndexReader reader = DirectoryReader.open(d)) {
+ BitSetProducer parentFilter =
+ new QueryBitSetProducer(new TermQuery(new Term("docType", "_parent")));
+ IndexSearcher searcher = newSearcher(reader);
+ for (int i = 0; i < numIters; i++) {
+ int k = random().nextInt(80) + 1;
+ // TODO: test with child filter
+ Query query =
+ getParentJoinKnnQuery("field", randomVector(dimension), null, k, parentFilter);
+ int n = random().nextInt(100) + 1;
+ TopDocs results = searcher.search(query, n);
+ int expected = Math.min(Math.min(n, k), numParentsWithChildren);
+ // we may get fewer results than requested if there are deletions, but this test doesn't
+ // test that
+ assert reader.hasDeletions() == false;
+ /*
+ if (expected != results.scoreDocs.length) {
+ for (ScoreDoc doc : results.scoreDocs) {
+ System.out.println("result id=" + reader.storedFields().document(doc.doc).get("id"));
+ }
+ }
+ ` */
+ assertEquals(expected, results.scoreDocs.length);
+ assertTrue(results.totalHits.value() >= results.scoreDocs.length);
+ // verify the results are in descending score order
+ float last = Float.MAX_VALUE;
+ for (ScoreDoc scoreDoc : results.scoreDocs) {
+ assertTrue(scoreDoc.score <= last);
+ last = scoreDoc.score;
+ }
+ }
+ }
+ }
+ }
}
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinByteKnnVectorQuery.java b/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinByteKnnVectorQuery.java
index 6c1d461d4bf..2f5a50b0d7e 100644
--- a/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinByteKnnVectorQuery.java
+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinByteKnnVectorQuery.java
@@ -35,6 +35,8 @@ import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.BytesRef;
public class TestParentBlockJoinByteKnnVectorQuery extends ParentBlockJoinKnnVectorQueryTestCase {
@@ -105,4 +107,23 @@ public class TestParentBlockJoinByteKnnVectorQuery extends ParentBlockJoinKnnVec
}
return query;
}
+
+ @Override
+ float[] randomVector(int dim) {
+ BytesRef v = TestUtil.randomBinaryTerm(random(), TestUtil.nextInt(random(), 1, 100));
+ // clip at -127 to avoid overflow
+ for (int i = v.offset; i < v.offset + v.length; i++) {
+ if (v.bytes[i] == -128) {
+ v.bytes[i] = -127;
+ }
+ }
+ assert v.offset == 0;
+ byte[] b = v.bytes;
+ float[] v1 = new float[b.length];
+ int vi = 0;
+ for (int i = 0; i < v.length; i++) {
+ v1[vi++] = b[i];
+ }
+ return v1;
+ }
}
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinFloatKnnVectorQuery.java b/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinFloatKnnVectorQuery.java
index 7b99b59c9a4..9a5837b15db 100644
--- a/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinFloatKnnVectorQuery.java
+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinFloatKnnVectorQuery.java
@@ -22,6 +22,7 @@ import static org.apache.lucene.index.VectorSimilarityFunction.COSINE;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Random;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.KnnFloatVectorField;
@@ -140,4 +141,14 @@ public class TestParentBlockJoinFloatKnnVectorQuery extends ParentBlockJoinKnnVe
String name, float[] vector, VectorSimilarityFunction vectorSimilarityFunction) {
return new KnnFloatVectorField(name, vector, vectorSimilarityFunction);
}
+
+ @Override
+ float[] randomVector(int dim) {
+ float[] v = new float[dim];
+ Random random = random();
+ for (int i = 0; i < dim; i++) {
+ v[i] = random.nextFloat();
+ }
+ return v;
+ }
}