Apache Lucene open-source search software
Add bulk-scorer differential oracle test (#16511)
Tim Allisonf7653e2a45c4
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex ee6598db13a..ce98146da8a 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -286,6 +286,8 @@ Other* GITHUB#16512: AssertingScorer's two-phase iterator now delegates docIDRunEnd(),so it exercises that contract in randomized tests. (Tim Allison)+* GITHUB#16511: Add bulk-scorer vs doc-by-doc differential oracle test. (Tim Allison)+* GITHUB#16074: Ban presized collection#toArray. (Zhou Hui)* GITHUB#16063: Remove redundant final modifiers. (Zhou Hui)diff --git a/lucene/core/src/test/org/apache/lucene/search/FieldSpec.java b/lucene/core/src/test/org/apache/lucene/search/FieldSpec.javanew file mode 100644index 00000000000..c2b007bd588--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/search/FieldSpec.java@@ -0,0 +1,42 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.search;++import java.util.List;+import org.apache.lucene.util.BytesRef;++/**+ * Shared schema description used by {@link RandomIndexBuilder} (write side) and {@link+ * RandomLuceneQueryGenerator} (query side).+ */+record FieldSpec(String name, FieldKind kind, List<BytesRef> terms, long minValue, long maxValue) {++ enum FieldKind {+ INDEXED_KEYWORD,+ DV_KEYWORD,+ DV_NUMERIC,+ POINT_LONG,+ }++ static FieldSpec keyword(String name, FieldKind kind, List<BytesRef> terms) {+ return new FieldSpec(name, kind, terms, 0, 0);+ }++ static FieldSpec numeric(String name, FieldKind kind, long minValue, long maxValue) {+ return new FieldSpec(name, kind, List.of(), minValue, maxValue);+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/search/RandomIndexBuilder.java b/lucene/core/src/test/org/apache/lucene/search/RandomIndexBuilder.javanew file mode 100644index 00000000000..45d3e01d299--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/search/RandomIndexBuilder.java@@ -0,0 +1,199 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.search;++import com.carrotsearch.randomizedtesting.generators.RandomPicks;+import java.io.IOException;+import java.util.ArrayList;+import java.util.List;+import java.util.Random;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.Field;+import org.apache.lucene.document.LongPoint;+import org.apache.lucene.document.SortedNumericDocValuesField;+import org.apache.lucene.document.SortedSetDocValuesField;+import org.apache.lucene.document.StringField;+import org.apache.lucene.index.Term;+import org.apache.lucene.search.FieldSpec.FieldKind;+import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.index.RandomIndexWriter;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.BytesRef;++/**+ * Builds a random Lucene index whose doc counts bracket the thresholds that gate bulk-scorer+ * specializations: the 4096-doc windows of the doc-values skip index, {@code+ * DenseConjunctionBulkScorer} and {@code BooleanScorer}, and the 1/32 density cutoff.+ */+class RandomIndexBuilder {++ static final int[] INTERESTING_DOC_COUNTS = interestingDocCounts();++ private static int[] interestingDocCounts() {+ int w = DenseConjunctionBulkScorer.WINDOW_SIZE;+ // BooleanScorer's window and the DV skip-index interval (Lucene90DocValuesFormat, private)+ // are the same size today; if this trips, add their boundaries separately.+ assert BooleanScorer.SIZE == w : BooleanScorer.SIZE;+ return new int[] {+ 0, 1, w / 8, w / 4, w / 2 - 1, w / 2, w / 2 + 1, w - 1, w, w + 1, 2 * w, 2 * w + 1+ };+ }++ private final Random random;++ /**+ * CYCLIC spreads the whole value domain through every skipper block (mostly MAYBE); CLUSTERED+ * gives blocks tight ranges (YES/NO, long runs); RANDOM sits in between.+ */+ private enum ValueLayout {+ CYCLIC,+ CLUSTERED,+ RANDOM+ }++ RandomIndexBuilder(Random random) {+ this.random = random;+ }++ private ValueLayout layout;+ private int numDocs;+ // one missing value in N docs; 0 = fully populated (count==maxDoc shortcuts, YES blocks)+ private int dvKeywordMissingOneIn;+ private int dvNumericMissingOneIn;++ List<FieldSpec> build(Directory dir) throws IOException {+ numDocs = chooseDocCount();+ layout = RandomPicks.randomFrom(random, ValueLayout.values());+ dvKeywordMissingOneIn = pickMissingOneIn();+ dvNumericMissingOneIn = pickMissingOneIn();+ // cardinality 1 = constant field (all blocks YES)+ int lowCard = TestUtil.nextInt(random, 1, 7);+ int highCard = Math.max(lowCard + 1, numDocs / 4);+ List<FieldSpec> specs = buildSpecs(lowCard, highCard);++ try (RandomIndexWriter w = new RandomIndexWriter(random, dir)) {+ for (int i = 0; i < numDocs; i++) {+ Document doc = new Document();+ doc.add(new StringField("id", Integer.toString(i), Field.Store.NO));+ for (FieldSpec spec : specs) {+ addFieldValue(doc, spec, i);+ }+ w.addDocument(doc);+ }++ if (random.nextBoolean() && numDocs > 1) {+ int toDelete = TestUtil.nextInt(random, 1, Math.min(numDocs, 10));+ for (int i = 0; i < toDelete; i++) {+ w.deleteDocuments(new Term("id", Integer.toString(random.nextInt(numDocs))));+ }+ }++ if (random.nextBoolean() && numDocs > 1) {+ // updates re-add docs at the end, decorrelating doc order from value patterns+ int toUpdate = TestUtil.nextInt(random, 1, Math.min(numDocs, 10));+ for (int i = 0; i < toUpdate; i++) {+ int id = random.nextInt(numDocs);+ Document doc = new Document();+ doc.add(new StringField("id", Integer.toString(id), Field.Store.NO));+ for (FieldSpec spec : specs) {+ addFieldValue(doc, spec, id);+ }+ w.updateDocument(new Term("id", Integer.toString(id)), doc);+ }+ }++ if (random.nextBoolean()) {+ w.forceMerge(1);+ }+ }++ return specs;+ }++ private int chooseDocCount() {+ if (random.nextInt(3) == 0) {+ return INTERESTING_DOC_COUNTS[random.nextInt(INTERESTING_DOC_COUNTS.length)];+ }+ return TestUtil.nextInt(random, 1, INTERESTING_DOC_COUNTS[INTERESTING_DOC_COUNTS.length - 1]);+ }++ private int pickMissingOneIn() {+ return switch (random.nextInt(3)) {+ case 0 -> 0; // present in every doc+ case 1 -> 5; // occasional gaps+ default -> 2; // half missing+ };+ }++ private boolean present(int missingOneIn) {+ return missingOneIn == 0 || random.nextInt(missingOneIn) != 0;+ }++ private List<FieldSpec> buildSpecs(int lowCard, int highCard) {+ List<FieldSpec> specs = new ArrayList<>();+ specs.add(FieldSpec.keyword("f_kw", FieldKind.INDEXED_KEYWORD, makeTerms("kw", lowCard)));+ specs.add(FieldSpec.keyword("f_dv_kw", FieldKind.DV_KEYWORD, makeTerms("dv", lowCard)));+ specs.add(FieldSpec.numeric("f_dv_num", FieldKind.DV_NUMERIC, 0L, highCard - 1L));+ specs.add(FieldSpec.numeric("f_pt", FieldKind.POINT_LONG, 0L, highCard - 1L));+ return specs;+ }++ private List<BytesRef> makeTerms(String prefix, int n) {+ List<BytesRef> terms = new ArrayList<>(n);+ for (int i = 0; i < n; i++) {+ terms.add(new BytesRef(prefix + "_" + i));+ }+ return terms;+ }++ private int ordinalFor(int docId, int n) {+ return switch (layout) {+ case CYCLIC -> docId % n;+ case CLUSTERED -> (int) ((long) docId * n / numDocs);+ case RANDOM -> random.nextInt(n);+ };+ }++ private void addFieldValue(Document doc, FieldSpec spec, int docId) {+ switch (spec.kind()) {+ case INDEXED_KEYWORD -> {+ BytesRef term = spec.terms().get(ordinalFor(docId, spec.terms().size()));+ doc.add(new StringField(spec.name(), term.utf8ToString(), Field.Store.NO));+ }+ case DV_KEYWORD -> {+ BytesRef term = spec.terms().get(ordinalFor(docId, spec.terms().size()));+ if (present(dvKeywordMissingOneIn)) {+ doc.add(SortedSetDocValuesField.indexedField(spec.name(), term));+ }+ }+ case DV_NUMERIC -> {+ long val =+ spec.minValue() + ordinalFor(docId, (int) (spec.maxValue() - spec.minValue() + 1));+ if (present(dvNumericMissingOneIn)) {+ // POINT_LONG's DV field below stays plain, covering the no-skipper variant+ doc.add(SortedNumericDocValuesField.indexedField(spec.name(), val));+ }+ }+ case POINT_LONG -> {+ long val =+ spec.minValue() + ordinalFor(docId, (int) (spec.maxValue() - spec.minValue() + 1));+ doc.add(new LongPoint(spec.name() + "_point", val));+ doc.add(new SortedNumericDocValuesField(spec.name() + "_dv", val));+ }+ }+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/search/RandomLuceneQueryGenerator.java b/lucene/core/src/test/org/apache/lucene/search/RandomLuceneQueryGenerator.javanew file mode 100644index 00000000000..a3c002dbada--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/search/RandomLuceneQueryGenerator.java@@ -0,0 +1,247 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.search;++import com.carrotsearch.randomizedtesting.generators.RandomPicks;+import java.util.ArrayList;+import java.util.List;+import java.util.Random;+import org.apache.lucene.document.LongPoint;+import org.apache.lucene.document.SortedNumericDocValuesField;+import org.apache.lucene.document.SortedSetDocValuesField;+import org.apache.lucene.index.Term;+import org.apache.lucene.search.BooleanClause.Occur;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.BytesRef;++/**+ * Generates random {@link Query} trees over a {@link FieldSpec} schema for differential oracle+ * testing. Driven by a caller-supplied {@link Random}; a failing seed replays exactly.+ */+class RandomLuceneQueryGenerator {++ static final int MAX_DEPTH = 3;++ private final Random random;+ private final List<FieldSpec> fields;+ // wide enough for a conjunction to touch every field kind plus a duplicate+ private final int maxClauses;++ RandomLuceneQueryGenerator(Random random, List<FieldSpec> fields) {+ this.random = random;+ this.fields = fields;+ this.maxClauses = Math.max(5, fields.size() + 1);+ }++ Query next() {+ // bias toward DenseConjunctionBulkScorer (all-FILTER conjunctions) and BooleanScorer's+ // bucketed path (wide flat disjunctions)+ if (random.nextInt(3) == 0) {+ return randomFilterConjunction();+ }+ if (random.nextInt(8) == 0) {+ return randomWideDisjunction();+ }+ return generate(MAX_DEPTH);+ }++ private Query randomFilterConjunction() {+ int numClauses = TestUtil.nextInt(random, 2, maxClauses);+ BooleanQuery.Builder b = new BooleanQuery.Builder();+ for (int i = 0; i < numClauses; i++) {+ b.add(randomLeaf(), Occur.FILTER);+ }+ return b.build();+ }++ private Query randomWideDisjunction() {+ int numClauses = TestUtil.nextInt(random, 2, 16);+ BooleanQuery.Builder b = new BooleanQuery.Builder();+ for (int i = 0; i < numClauses; i++) {+ b.add(randomLeaf(), Occur.SHOULD);+ }+ b.setMinimumNumberShouldMatch(random.nextInt(3));+ return b.build();+ }++ private Query generate(int depth) {+ // 30% early-out below the root tapers tree depth+ if (depth == 0 || (depth < MAX_DEPTH && random.nextInt(10) < 3)) {+ return randomLeaf();+ }+ return randomBool(depth);+ }++ private Query randomBool(int depth) {+ int numClauses = TestUtil.nextInt(random, 1, maxClauses);+ BooleanQuery.Builder b = new BooleanQuery.Builder();++ int mustNotCount = 0;+ List<Query> clauses = new ArrayList<>(numClauses);+ List<Occur> occurs = new ArrayList<>(numClauses);++ for (int i = 0; i < numClauses; i++) {+ clauses.add(generate(depth - 1));+ occurs.add(RandomPicks.randomFrom(random, Occur.values()));+ if (occurs.get(i) == Occur.MUST_NOT) {+ mustNotCount++;+ }+ }++ if (mustNotCount == numClauses) {+ occurs.set(random.nextInt(numClauses), Occur.FILTER);+ }++ int numShould = 0;+ for (int i = 0; i < numClauses; i++) {+ b.add(clauses.get(i), occurs.get(i));+ if (occurs.get(i) == Occur.SHOULD) {+ numShould++;+ }+ }++ if (numShould > 0) {+ b.setMinimumNumberShouldMatch(random.nextInt(numShould + 1));+ }++ return b.build();+ }++ private Query randomLeaf() {+ int roll = random.nextInt(20);+ if (roll == 0) return new MatchAllDocsQuery();+ if (roll == 1) return new MatchNoDocsQuery();+ if (roll == 2) return new ConstantScoreQuery(randomLeafFromField());+ return randomLeafFromField();+ }++ private Query randomLeafFromField() {+ FieldSpec f = RandomPicks.randomFrom(random, fields);+ return switch (f.kind()) {+ case INDEXED_KEYWORD -> randomIndexedKeywordLeaf(f);+ case DV_KEYWORD -> randomDvKeywordLeaf(f);+ case DV_NUMERIC -> randomDvNumericLeaf(f);+ case POINT_LONG -> randomPointLongLeaf(f);+ };+ }++ private Query randomIndexedKeywordLeaf(FieldSpec f) {+ return switch (random.nextInt(3)) {+ case 0 -> new TermQuery(new Term(f.name(), randomTerm(f)));+ case 1 -> randomTermInSet(f);+ case 2 -> randomTermRange(f);+ default -> throw new AssertionError();+ };+ }++ private Query randomTermInSet(FieldSpec f) {+ int n = TestUtil.nextInt(random, 1, Math.min(3, f.terms().size()));+ List<BytesRef> set = new ArrayList<>(n);+ for (int i = 0; i < n; i++) {+ set.add(randomTerm(f));+ }+ return new TermInSetQuery(f.name(), set);+ }++ private Query randomTermRange(FieldSpec f) {+ BytesRef lo = randomTermOrNull(f);+ BytesRef hi = randomTermOrNull(f);+ if (lo != null && hi != null && lo.compareTo(hi) > 0) {+ BytesRef tmp = lo;+ lo = hi;+ hi = tmp;+ }+ return new TermRangeQuery(f.name(), lo, hi, random.nextBoolean(), random.nextBoolean());+ }++ private Query randomDvKeywordLeaf(FieldSpec f) {+ return switch (random.nextInt(3)) {+ case 0 -> SortedSetDocValuesField.newSlowExactQuery(f.name(), randomTerm(f));+ case 1 -> randomDvKeywordRangeLeaf(f);+ case 2 -> randomDvKeywordSetLeaf(f);+ default -> throw new AssertionError();+ };+ }++ private Query randomDvKeywordRangeLeaf(FieldSpec f) {+ BytesRef a = randomTerm(f);+ BytesRef b = randomTerm(f);+ boolean ordered = a.compareTo(b) <= 0;+ return SortedSetDocValuesField.newSlowRangeQuery(+ f.name(), ordered ? a : b, ordered ? b : a, true, true);+ }++ /**+ * Picks every other term so there is always an ordinal gap, forcing the non-contiguous ordinal+ * set iterator whose {@code docIDRunEnd()} had the GH#16450 bug.+ */+ private Query randomDvKeywordSetLeaf(FieldSpec f) {+ List<BytesRef> all = f.terms();+ if (all.size() < 3) {+ return SortedSetDocValuesField.newSlowExactQuery(f.name(), randomTerm(f));+ }+ int start = random.nextInt(2);+ List<BytesRef> subset = new ArrayList<>();+ for (int i = start; i < all.size(); i += 2) {+ subset.add(all.get(i));+ }+ if (subset.size() < 2) {+ subset.add(all.get(all.size() - 1));+ }+ return SortedSetDocValuesField.newSlowSetQuery(f.name(), subset);+ }++ private Query randomDvNumericLeaf(FieldSpec f) {+ if (random.nextBoolean()) {+ return SortedNumericDocValuesField.newSlowExactQuery(f.name(), randomLong(f));+ }+ long a = randomLong(f);+ long b = randomLong(f);+ return SortedNumericDocValuesField.newSlowRangeQuery(f.name(), Math.min(a, b), Math.max(a, b));+ }++ private Query randomPointLongLeaf(FieldSpec f) {+ long a = randomLong(f);+ long b = randomLong(f);+ long lo = Math.min(a, b);+ long hi = Math.max(a, b);+ Query pointQuery = LongPoint.newRangeQuery(f.name() + "_point", lo, hi);+ Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(f.name() + "_dv", lo, hi);+ if (random.nextBoolean()) {+ return new IndexOrDocValuesQuery(pointQuery, dvQuery);+ }+ return random.nextBoolean() ? pointQuery : dvQuery;+ }++ private BytesRef randomTerm(FieldSpec f) {+ if (random.nextInt(5) == 0 || f.terms().isEmpty()) {+ return new BytesRef("zzz_outside_" + random.nextInt(100));+ }+ return RandomPicks.randomFrom(random, f.terms());+ }++ private BytesRef randomTermOrNull(FieldSpec f) {+ return random.nextInt(5) == 0 ? null : randomTerm(f);+ }++ private long randomLong(FieldSpec f) {+ if (random.nextInt(5) == 0) {+ return f.maxValue() + TestUtil.nextInt(random, 1, 10);+ }+ return TestUtil.nextLong(random, f.minValue(), f.maxValue());+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBulkScorerConsistency.java b/lucene/core/src/test/org/apache/lucene/search/TestBulkScorerConsistency.javanew file mode 100644index 00000000000..c357d21261c--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/search/TestBulkScorerConsistency.java@@ -0,0 +1,235 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.search;++import java.io.IOException;+import java.util.List;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.Field;+import org.apache.lucene.document.SortedDocValuesField;+import org.apache.lucene.document.StringField;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.LeafReaderContext;+import org.apache.lucene.index.Term;+import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.search.FixedBitSetCollector;+import org.apache.lucene.tests.search.ScorerIndexSearcher;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.Bits;+import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.FixedBitSet;++/**+ * Differential oracle: the optimised bulk-scoring path must produce the same document set as the+ * doc-by-doc {@link Scorer} path ({@link ScorerIndexSearcher}) for random queries over random+ * indexes. Compares match sets only, not scores. Written after the {@code+ * DocValuesRangeIterator.docIDRunEnd()} bug (GH#16450), which this harness catches.+ */+public class TestBulkScorerConsistency extends LuceneTestCase {++ /**+ * Random index, random queries, fast == slow. The fast side is a bare {@link IndexSearcher} so+ * the production bulk scorers run unwrapped by the test framework.+ */+ public void testBulkEqualsScorer() throws IOException {+ try (Directory dir = newDirectory()) {+ List<FieldSpec> fields = new RandomIndexBuilder(random()).build(dir);+ try (DirectoryReader reader = DirectoryReader.open(dir)) {+ IndexSearcher fastSearcher = new IndexSearcher(reader);+ ScorerIndexSearcher slowSearcher = new ScorerIndexSearcher(reader);+ fastSearcher.setQueryCache(null);+ slowSearcher.setQueryCache(null);++ RandomLuceneQueryGenerator gen = new RandomLuceneQueryGenerator(random(), fields);+ int numQueries = TestUtil.nextInt(random(), 20, 100);+ for (int q = 0; q < numQueries; q++) {+ assertBulkEqualsScorer(fastSearcher, slowSearcher, gen.next());+ }+ }+ }+ }++ /** Self-test: a bulk scorer that collects without checking matches must be flagged. */+ public void testHarnessDetectsBrokenBulkScorer() throws IOException {+ try (Directory dir = newDirectory()) {+ try (IndexWriter w = new IndexWriter(dir, new IndexWriterConfig())) {+ for (int i = 0; i < 100; i++) {+ Document doc = new Document();+ doc.add(new StringField("f", i % 2 == 0 ? "even" : "odd", Field.Store.NO));+ w.addDocument(doc);+ }+ }++ try (DirectoryReader reader = DirectoryReader.open(dir)) {+ ScorerIndexSearcher correct = new ScorerIndexSearcher(reader);+ LyingIndexSearcher lying = new LyingIndexSearcher(reader);+ correct.setQueryCache(null);+ lying.setQueryCache(null);++ int maxDoc = reader.maxDoc();+ Query query = new TermQuery(new Term("f", "even"));++ FixedBitSet correctResult =+ correct.search(query, FixedBitSetCollector.createManager(maxDoc));+ FixedBitSet lyingResult = lying.search(query, FixedBitSetCollector.createManager(maxDoc));++ assertEquals(50, correctResult.cardinality());+ assertFalse(lyingResult.equals(correctResult));+ assertEquals(maxDoc, lyingResult.cardinality());+ }+ }+ }++ /**+ * Regression test for GH#16450: {@code docIDRunEnd()} over-reported for a non-contiguous ordinal+ * set, so on a {@code WINDOW_SIZE + 1}-doc index the trailing single-doc window looked fully+ * matching and {@code DenseConjunctionBulkScorer} collected it via {@code collectRange} without+ * confirming {@code matches()}.+ */+ public void testDVOrdinalSetFalsePositive() throws IOException {+ int maxDoc = DenseConjunctionBulkScorer.WINDOW_SIZE + 1;+ try (Directory dir = newDirectory()) {+ try (IndexWriter w = new IndexWriter(dir, new IndexWriterConfig())) {+ for (int i = 0; i < maxDoc; i++) {+ Document doc = new Document();+ // Most docs: "aaa" (ord 0). Every 100th: "ccc" (ord 2) to establish the 3-term vocab.+ // Last doc: "bbb" (ord 1) — inside bounding range [0,2] but NOT in set {0,2}.+ String val = (i == maxDoc - 1) ? "bbb" : (i % 100 == 0 ? "ccc" : "aaa");+ doc.add(SortedDocValuesField.indexedField("dv", new BytesRef(val)));+ w.addDocument(doc);+ }+ w.forceMerge(1);+ }+ try (DirectoryReader reader = DirectoryReader.open(dir)) {+ IndexSearcher fast = new IndexSearcher(reader);+ ScorerIndexSearcher slow = new ScorerIndexSearcher(reader);+ fast.setQueryCache(null);+ slow.setQueryCache(null);++ // Ordinal set {aaa=0, ccc=2} with gap at bbb=1: non-contiguous, so the block iterator+ // cannot prove whole-block matches and docIDRunEnd() must stay conservative.+ Query setQuery =+ SortedDocValuesField.newSlowSetQuery(+ "dv", List.of(new BytesRef("aaa"), new BytesRef("ccc")));+ Query rangeQuery =+ SortedDocValuesField.newSlowRangeQuery(+ "dv", new BytesRef("aaa"), new BytesRef("ccc"), true, true);+ Query q =+ new BooleanQuery.Builder()+ .add(setQuery, BooleanClause.Occur.FILTER)+ .add(rangeQuery, BooleanClause.Occur.FILTER)+ .build();++ FixedBitSet fastBits = fast.search(q, FixedBitSetCollector.createManager(maxDoc));+ FixedBitSet slowBits = slow.search(q, FixedBitSetCollector.createManager(maxDoc));+ assertEquals("fast path cardinality", maxDoc - 1, fastBits.cardinality());+ assertEquals("slow path cardinality", maxDoc - 1, slowBits.cardinality());+ }+ }+ }++ static void assertBulkEqualsScorer(+ IndexSearcher fastSearcher, ScorerIndexSearcher slowSearcher, Query query)+ throws IOException {+ int maxDoc = fastSearcher.getIndexReader().maxDoc();+ FixedBitSet fast = fastSearcher.search(query, FixedBitSetCollector.createManager(maxDoc));+ FixedBitSet slow = slowSearcher.search(query, FixedBitSetCollector.createManager(maxDoc));+ if (slow.equals(fast) == false) {+ StringBuilder diff = new StringBuilder();+ for (int doc = 0, shown = 0; doc < maxDoc && shown < 10; doc++) {+ if (slow.get(doc) != fast.get(doc)) {+ diff.append(doc).append(slow.get(doc) ? " missed by bulk; " : " bulk false positive; ");+ shown++;+ }+ }+ fail("bulk scorer and doc-by-doc scorer disagree on: " + query + " — docs: " + diff);+ }+ }++ // Replaces the real bulk scorer with one that collects every doc in [min, max)+ // unconditionally (i.e. claims all docs match). Used only by testHarnessDetectsBrokenBulkScorer+ // to confirm the oracle catches a scorer that over-reports.+ static class LyingIndexSearcher extends IndexSearcher {+ LyingIndexSearcher(org.apache.lucene.index.IndexReader r) {+ super(r);+ }++ @Override+ protected void searchLeaf(+ LeafReaderContext ctx, int minDocId, int maxDocId, Weight weight, Collector collector)+ throws IOException {+ final LeafCollector leafCollector;+ try {+ leafCollector = collector.getLeafCollector(ctx);+ } catch (CollectionTerminatedException _) {+ return;+ }+ ScorerSupplier ss = weight.scorerSupplier(ctx);+ if (ss == null) {+ leafCollector.finish();+ return;+ }+ BulkScorer real = ss.bulkScorer();+ if (real == null) {+ leafCollector.finish();+ return;+ }+ try {+ int maxDoc = Math.min(maxDocId, ctx.reader().maxDoc());+ new LyingBulkScorer(real)+ .score(leafCollector, ctx.reader().getLiveDocs(), minDocId, maxDoc);+ } catch (CollectionTerminatedException _) {+ // normal+ }+ leafCollector.finish();+ }+ }++ static class LyingBulkScorer extends BulkScorer {+ private final BulkScorer in;++ LyingBulkScorer(BulkScorer in) {+ this.in = in;+ }++ @Override+ public int score(LeafCollector collector, Bits acceptDocs, int min, int max)+ throws IOException {+ collector.setScorer(+ new Scorable() {+ @Override+ public float score() {+ return 0f;+ }+ });+ for (int doc = min; doc < max; doc++) {+ if (acceptDocs == null || acceptDocs.get(doc)) {+ collector.collect(doc);+ }+ }+ return max;+ }++ @Override+ public long cost() {+ return in.cost();+ }+ }+}