Apache Lucene open-source search software
LUCENE-10530: Avoid floating point precision bug in TestTaxonomyFacetAssociations (#848)
Greg Miller902a7df0e500
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 956200420b3..2e580e89a77 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -80,7 +80,7 @@ New Featuresto speed up computing the number of hits when possible. (Lu Xugang, Luca Cavanna, Adrien Grand)* LUCENE-10422: Monitor Improvements: `Monitor` can use a custom `Directory`- implementation. `Monitor` can be created with a readonly `QueryIndex` in order to+ implementation. `Monitor` can be created with a readonly `QueryIndex` in order tohave readonly `Monitor` instances. (Niko Usai)* LUCENE-10456: Implement rewrite and Weight#count for MultiRangeQuery@@ -153,8 +153,11 @@ Bug Fixesno documents instead of throwing an NPE. (Greg Miller)* LUCENE-10470: Check if polygon has been successfully tessellated before we fail (we are failing some valid- tessellations) and allow filtering edges that fold on top of the previous one. (Ignacio Vera)-+ tessellations) and allow filtering edges that fold on top of the previous one. (Ignacio Vera)++* LUCENE-10530: Avoid floating point precision test case bug in TestTaxonomyFacetAssociations.+ (Greg Miller)+Build---------------------diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetAssociations.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetAssociations.javaindex eb6088922eb..9cb9963e315 100644--- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetAssociations.java+++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetAssociations.java@@ -31,11 +31,17 @@ import org.apache.lucene.facet.FacetsCollectorManager;import org.apache.lucene.facet.FacetsConfig;import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;+import org.apache.lucene.index.BinaryDocValues;+import org.apache.lucene.index.DocValues;import org.apache.lucene.index.IndexReader;+import org.apache.lucene.index.LeafReaderContext;+import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.MatchAllDocsQuery;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.index.RandomIndexWriter;+import org.apache.lucene.util.BitUtil;+import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.IOUtils;import org.junit.AfterClass;import org.junit.BeforeClass;@@ -115,7 +121,6 @@ public class TestTaxonomyFacetAssociations extends FacetTestCase {}if (random().nextBoolean()) { // maybe index a float association with the dimfloat nextFloat = random().nextFloat() * 10000f;- randomFloatValues.computeIfAbsent(path, k -> new ArrayList<>()).add(nextFloat);doc.add(new FloatAssociationFacetField(nextFloat, "float_random", path));}}@@ -129,7 +134,6 @@ public class TestTaxonomyFacetAssociations extends FacetTestCase {}if (random().nextBoolean()) {float nextFloat = random().nextFloat() * 10000f;- randomFloatSingleValued.computeIfAbsent(path, k -> new ArrayList<>()).add(nextFloat);doc.add(new FloatAssociationFacetField(nextFloat, "float_single_valued", path));}}@@ -141,6 +145,34 @@ public class TestTaxonomyFacetAssociations extends FacetTestCase {reader = writer.getReader();writer.close();taxoReader = new DirectoryTaxonomyReader(taxoDir);++ // To avoid floating point precision issues, it's useful to keep track of the values in the+ // exact same order they appear when iterating the doc values in the index. This ensures we+ // sum them in the same order when determining expected values for tests cases and when the+ // actual facets implementation sums them. See LUCENE-10530:+ for (LeafReaderContext ctx : reader.leaves()) {+ BinaryDocValues dv = DocValues.getBinary(ctx.reader(), "$facets.float");+ for (int doc = dv.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = dv.nextDoc()) {+ final BytesRef bytesRef = dv.binaryValue();+ byte[] bytes = bytesRef.bytes;+ int end = bytesRef.offset + bytesRef.length;+ int offset = bytesRef.offset;+ while (offset < end) {+ int ord = (int) BitUtil.VH_BE_INT.get(bytes, offset);+ offset += 4;+ float value = (float) BitUtil.VH_BE_FLOAT.get(bytes, offset);+ offset += 4;+ FacetLabel label = taxoReader.getPath(ord);+ String dim = label.components[0];+ String child = label.components[1];+ if ("float_random".equals(dim)) {+ randomFloatValues.computeIfAbsent(child, k -> new ArrayList<>()).add(value);+ } else if ("float_single_valued".equals(dim)) {+ randomFloatSingleValued.computeIfAbsent(child, k -> new ArrayList<>()).add(value);+ }+ }+ }+ }}@AfterClass@@ -449,7 +481,10 @@ public class TestTaxonomyFacetAssociations extends FacetTestCase {float aggregatedValue = 0f;for (Map.Entry<String, Float> e : expected.entrySet()) {float value = e.getValue();- assertEquals(value, facets.getSpecificValue(dim, e.getKey()).floatValue(), 1);+ // We can expect the floats to be exactly equal here since we're ensuring that we sum them+ // in the same order when determining expected values and when computing facets. See+ // LUCENE-10530:+ assertEquals(value, facets.getSpecificValue(dim, e.getKey()).floatValue(), 0f);aggregatedValue = aggregationFunction.aggregate(aggregatedValue, value);}@@ -465,7 +500,7 @@ public class TestTaxonomyFacetAssociations extends FacetTestCase {assertNull(facetResult);} else {assertEquals(dim, facetResult.dim);- assertEquals(aggregatedValue, facetResult.value.floatValue(), 1);+ assertEquals(aggregatedValue, facetResult.value.floatValue(), 1f);assertEquals(expected.size(), facetResult.childCount);}}