Apache Lucene open-source search software
Add ParentBlockJoinKnnVectorQueryTestCase.testRandom
Michael Sokolov76affdc78fcc
diff --git a/lucene/join/src/test/org/apache/lucene/search/join/ParentBlockJoinKnnVectorQueryTestCase.java b/lucene/join/src/test/org/apache/lucene/search/join/ParentBlockJoinKnnVectorQueryTestCase.javaindex 2594c20ab8e..023a5cd3ea2 100644--- a/lucene/join/src/test/org/apache/lucene/search/join/ParentBlockJoinKnnVectorQueryTestCase.java+++ b/lucene/join/src/test/org/apache/lucene/search/join/ParentBlockJoinKnnVectorQueryTestCase.java@@ -28,6 +28,7 @@ import java.util.stream.Collectors;import java.util.stream.IntStream;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;+import org.apache.lucene.document.StoredField;import org.apache.lucene.document.StringField;import org.apache.lucene.index.DirectoryReader;import org.apache.lucene.index.IndexReader;@@ -73,6 +74,8 @@ abstract class ParentBlockJoinKnnVectorQueryTestCase extends LuceneTestCase {return parent;}+ abstract float[] randomVector(int dim);+abstract Query getParentJoinKnnQuery(String fieldName, float[] queryVector, Query childFilter, int k, BitSetProducer parentBitSet);@@ -108,8 +111,7 @@ abstract class ParentBlockJoinKnnVectorQueryTestCase extends LuceneTestCase {try (IndexReader reader = DirectoryReader.open(d)) {IndexSearcher searcher = new IndexSearcher(reader);// Create parent filter directly, tests use "check" to verify parentIds exist. Production- // may not- // verify we handle it gracefully+ // may not verify we handle it gracefullyBitSetProducer parentFilter =new QueryBitSetProducer(new TermQuery(new Term("docType", "_parent")));Query query = getParentJoinKnnQuery("field", new float[] {2, 2}, null, 3, parentFilter);@@ -405,4 +407,73 @@ abstract class ParentBlockJoinKnnVectorQueryTestCase extends LuceneTestCase {return true;}}++ /** Tests with random vectors, number of documents, etc. Uses RandomIndexWriter. */+ public void testRandom() throws IOException {+ int numDocs = atLeast(100);+ int dimension = atLeast(5);+ int numIters = atLeast(10);+ boolean everyDocHasAVector = random().nextBoolean();+ int numParentsWithChildren = 0;+ try (Directory d = newDirectory()) {+ RandomIndexWriter w = new RandomIndexWriter(random(), d);+ for (int i = 0; i < numDocs; i++) {+ List<Document> family = new ArrayList<>();+ Document doc = new Document();+ if (random().nextInt(5) == 1) {+ if (family.isEmpty() == false) {+ ++numParentsWithChildren;+ // System.out.println("parent w/children id=" + i);+ doc.add(new StoredField("id", Integer.toString(i)));+ } else {+ doc.add(new StoredField("id", "pnoc" + Integer.toString(i)));+ }+ doc.add(new StringField("docType", "_parent", Field.Store.NO));+ family.add(doc);+ w.addDocuments(family);+ family.clear();+ } else if (everyDocHasAVector || random().nextInt(10) != 2) {+ // NOTE: only child documents are allowed to have a vector!+ // Otherwise the query's assumptions are invalidated??+ doc.add(getKnnVectorField("field", randomVector(dimension)));+ doc.add(new StoredField("id", "c" + Integer.toString(i)));+ family.add(doc);+ }+ }+ w.close();+ // trailing children with no parent document are dropped+ try (IndexReader reader = DirectoryReader.open(d)) {+ BitSetProducer parentFilter =+ new QueryBitSetProducer(new TermQuery(new Term("docType", "_parent")));+ IndexSearcher searcher = newSearcher(reader);+ for (int i = 0; i < numIters; i++) {+ int k = random().nextInt(80) + 1;+ // TODO: test with child filter+ Query query =+ getParentJoinKnnQuery("field", randomVector(dimension), null, k, parentFilter);+ int n = random().nextInt(100) + 1;+ TopDocs results = searcher.search(query, n);+ int expected = Math.min(Math.min(n, k), numParentsWithChildren);+ // we may get fewer results than requested if there are deletions, but this test doesn't+ // test that+ assert reader.hasDeletions() == false;+ /*+ if (expected != results.scoreDocs.length) {+ for (ScoreDoc doc : results.scoreDocs) {+ System.out.println("result id=" + reader.storedFields().document(doc.doc).get("id"));+ }+ }+ ` */+ assertEquals(expected, results.scoreDocs.length);+ assertTrue(results.totalHits.value() >= results.scoreDocs.length);+ // verify the results are in descending score order+ float last = Float.MAX_VALUE;+ for (ScoreDoc scoreDoc : results.scoreDocs) {+ assertTrue(scoreDoc.score <= last);+ last = scoreDoc.score;+ }+ }+ }+ }+ }}diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinByteKnnVectorQuery.java b/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinByteKnnVectorQuery.javaindex 6c1d461d4bf..2f5a50b0d7e 100644--- a/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinByteKnnVectorQuery.java+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinByteKnnVectorQuery.java@@ -35,6 +35,8 @@ import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.Query;import org.apache.lucene.search.TermQuery;import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.BytesRef;public class TestParentBlockJoinByteKnnVectorQuery extends ParentBlockJoinKnnVectorQueryTestCase {@@ -105,4 +107,23 @@ public class TestParentBlockJoinByteKnnVectorQuery extends ParentBlockJoinKnnVec}return query;}++ @Override+ float[] randomVector(int dim) {+ BytesRef v = TestUtil.randomBinaryTerm(random(), TestUtil.nextInt(random(), 1, 100));+ // clip at -127 to avoid overflow+ for (int i = v.offset; i < v.offset + v.length; i++) {+ if (v.bytes[i] == -128) {+ v.bytes[i] = -127;+ }+ }+ assert v.offset == 0;+ byte[] b = v.bytes;+ float[] v1 = new float[b.length];+ int vi = 0;+ for (int i = 0; i < v.length; i++) {+ v1[vi++] = b[i];+ }+ return v1;+ }}diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinFloatKnnVectorQuery.java b/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinFloatKnnVectorQuery.javaindex 7b99b59c9a4..9a5837b15db 100644--- a/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinFloatKnnVectorQuery.java+++ b/lucene/join/src/test/org/apache/lucene/search/join/TestParentBlockJoinFloatKnnVectorQuery.java@@ -22,6 +22,7 @@ import static org.apache.lucene.index.VectorSimilarityFunction.COSINE;import java.io.IOException;import java.util.ArrayList;import java.util.List;+import java.util.Random;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.document.KnnFloatVectorField;@@ -140,4 +141,14 @@ public class TestParentBlockJoinFloatKnnVectorQuery extends ParentBlockJoinKnnVeString name, float[] vector, VectorSimilarityFunction vectorSimilarityFunction) {return new KnnFloatVectorField(name, vector, vectorSimilarityFunction);}++ @Override+ float[] randomVector(int dim) {+ float[] v = new float[dim];+ Random random = random();+ for (int i = 0; i < dim; i++) {+ v[i] = random.nextFloat();+ }+ return v;+ }}