Skip to content

Apache Lucene open-source search software

Remove duplicates from the hnsw recall testing (#14234)

Benjamin Trentaaa4a204dd12
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
index 97b578e7c5c..3e5983ce9ce 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java
@@ -1952,7 +1952,7 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
// indexed 421 lines from LICENSE.txt
// indexed 157 lines from NOTICE.txt
int topK = 10;
- int numQueries = 578;
+ int numQueries = 526;
String[] testQueries = {
"Apache Lucene",
"Apache License",
@@ -2036,6 +2036,7 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
Directory indexStore = newDirectory(random());
IndexWriter writer = new IndexWriter(indexStore, newIndexWriterConfig());
float[] scratch = new float[dimension];
+ Set<String> seen = new HashSet<>(578);
for (String file : List.of("LICENSE.txt", "NOTICE.txt")) {
try (InputStream in = BaseKnnVectorsFormatTestCase.class.getResourceAsStream(file);
BufferedReader reader = new BufferedReader(new InputStreamReader(in, UTF_8))) {
@@ -2046,6 +2047,9 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe
if (line.isEmpty()) {
continue;
}
+ if (seen.add(line) == false) {
+ continue;
+ }
++lineNo;
Document doc = new Document();
doc.add(