Skip to content

Apache Lucene open-source search software

LUCENE-10488: Optimized getTopDims in ConcurrentSSDVFacetCounts (#777)

Yuting Ganef43242d77a2
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/sortedset/ConcurrentSortedSetDocValuesFacetCounts.java b/lucene/facet/src/java/org/apache/lucene/facet/sortedset/ConcurrentSortedSetDocValuesFacetCounts.java
index d28182bc36d..59cef70ebfb 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/sortedset/ConcurrentSortedSetDocValuesFacetCounts.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/sortedset/ConcurrentSortedSetDocValuesFacetCounts.java
@@ -21,6 +21,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.List;
import java.util.PrimitiveIterator;
import java.util.concurrent.Callable;
@@ -54,6 +55,7 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LongValues;
+import org.apache.lucene.util.PriorityQueue;
/**
* Like {@link SortedSetDocValuesFacetCounts}, but aggregates counts concurrently across segments.
@@ -131,6 +133,10 @@ public class ConcurrentSortedSetDocValuesFacetCounts extends Facets {
}
}
+ /**
+ * Overloaded method to allow getPathResult be called without passing in the dimToChildOrdsResult
+ * parameter
+ */
private FacetResult getPathResult(
FacetsConfig.DimConfig dimConfig,
String dim,
@@ -139,16 +145,59 @@ public class ConcurrentSortedSetDocValuesFacetCounts extends Facets {
PrimitiveIterator.OfInt childOrds,
int topN)
throws IOException {
+ return getPathResult(dimConfig, dim, path, pathOrd, childOrds, topN, null);
+ }
- TopOrdAndIntQueue q = null;
+ /** Returns path results for a dimension */
+ private FacetResult getPathResult(
+ FacetsConfig.DimConfig dimConfig,
+ String dim,
+ String[] path,
+ int pathOrd,
+ PrimitiveIterator.OfInt childOrds,
+ int topN,
+ ChildOrdsResult dimToChildOrdsResult)
+ throws IOException {
- int bottomCount = 0;
+ ChildOrdsResult childOrdsResult;
+
+ // if getTopDims is called, get results from previously stored dimToChildOrdsResult, otherwise
+ // call getChildOrdsResult to get dimCount, childCount and the queue for the dimension's top
+ // children
+ if (dimToChildOrdsResult != null) {
+ childOrdsResult = dimToChildOrdsResult;
+ } else {
+ childOrdsResult = getChildOrdsResult(childOrds, topN, dimConfig, pathOrd);
+ }
+
+ if (childOrdsResult.q == null) {
+ return null;
+ }
+
+ LabelAndValue[] labelValues = getLabelValuesFromTopOrdAndIntQueue(childOrdsResult.q);
+ if (dimConfig.hierarchical == true) {
+ return new FacetResult(
+ dim, path, childOrdsResult.dimCount, labelValues, childOrdsResult.childCount);
+ } else {
+ return new FacetResult(
+ dim, emptyPath, childOrdsResult.dimCount, labelValues, childOrdsResult.childCount);
+ }
+ }
+
+ /**
+ * Returns ChildOrdsResult that contains results of dimCount, childCount, and the queue for the
+ * dimension's top children to populate FacetResult in getPathResult.
+ */
+ private ChildOrdsResult getChildOrdsResult(
+ PrimitiveIterator.OfInt childOrds, int topN, FacetsConfig.DimConfig dimConfig, int pathOrd) {
+
+ TopOrdAndIntQueue q = null;
+ int bottomCount = 0;
int dimCount = 0;
int childCount = 0;
TopOrdAndIntQueue.OrdAndValue reuse = null;
-
while (childOrds.hasNext()) {
int ord = childOrds.next();
if (counts.get(ord) > 0) {
@@ -173,10 +222,25 @@ public class ConcurrentSortedSetDocValuesFacetCounts extends Facets {
}
}
- if (q == null) {
- return null;
+ if (dimConfig.hierarchical == true) {
+ dimCount = counts.get(pathOrd);
+ } else {
+ // see if dimCount is actually reliable or needs to be reset
+ if (dimConfig.multiValued) {
+ if (dimConfig.requireDimCount) {
+ dimCount = counts.get(pathOrd);
+ } else {
+ dimCount = -1; // dimCount is in accurate at this point, so set it to -1
+ }
+ }
}
+ return new ChildOrdsResult(dimCount, childCount, q);
+ }
+
+ /** Returns label values for dims. */
+ private LabelAndValue[] getLabelValuesFromTopOrdAndIntQueue(TopOrdAndIntQueue q)
+ throws IOException {
LabelAndValue[] labelValues = new LabelAndValue[q.size()];
for (int i = labelValues.length - 1; i >= 0; i--) {
TopOrdAndIntQueue.OrdAndValue ordAndValue = q.pop();
@@ -185,20 +249,31 @@ public class ConcurrentSortedSetDocValuesFacetCounts extends Facets {
String[] parts = FacetsConfig.stringToPath(term.utf8ToString());
labelValues[i] = new LabelAndValue(parts[parts.length - 1], ordAndValue.value);
}
+ return labelValues;
+ }
- if (dimConfig.hierarchical == false) {
- // see if dimCount is actually reliable or needs to be reset
- if (dimConfig.multiValued) {
- if (dimConfig.requireDimCount) {
- dimCount = counts.get(pathOrd);
- } else {
- dimCount = -1; // dimCount is in accurate at this point, so set it to -1
- }
- }
- return new FacetResult(dim, emptyPath, dimCount, labelValues, childCount);
- } else {
- return new FacetResult(dim, path, counts.get(pathOrd), labelValues, childCount);
+ /** Returns value/count of a dimension. */
+ private int getDimValue(
+ FacetsConfig.DimConfig dimConfig,
+ String dim,
+ int dimOrd,
+ PrimitiveIterator.OfInt childOrds,
+ int topN,
+ HashMap<String, ChildOrdsResult> dimToChildOrdsResult) {
+
+ // if dimConfig.hierarchical == true || dim is multiValued and dim count has been aggregated at
+ // indexing time, return dimCount directly
+ if (dimConfig.hierarchical == true || (dimConfig.multiValued && dimConfig.requireDimCount)) {
+ return counts.get(dimOrd);
}
+
+ // if dimCount was not aggregated at indexing time, iterate over childOrds to get dimCount
+ ChildOrdsResult childOrdsResult = getChildOrdsResult(childOrds, topN, dimConfig, dimOrd);
+
+ // if no early termination, store dim and childOrdsResult into a hashmap to avoid calling
+ // getChildOrdsResult again in getPathResult
+ dimToChildOrdsResult.put(dim, childOrdsResult);
+ return childOrdsResult.dimCount;
}
private class CountOneSegment implements Callable<Void> {
@@ -474,33 +549,54 @@ public class ConcurrentSortedSetDocValuesFacetCounts extends Facets {
return counts.get(ord);
}
+ /**
+ * Overloaded method to allow getFacetResultForDim be called without passing in the
+ * dimToChildOrdsResult parameter
+ */
+ private FacetResult getFacetResultForDim(String dim, int topNChildren) throws IOException {
+ return getFacetResultForDim(dim, topNChildren, null);
+ }
+
+ /** Returns FacetResult for a dimension. */
+ private FacetResult getFacetResultForDim(
+ String dim, int topNChildren, ChildOrdsResult dimToChildOrdsResult) throws IOException {
+
+ FacetsConfig.DimConfig dimConfig = stateConfig.getDimConfig(dim);
+
+ if (dimConfig.hierarchical) {
+ SortedSetDocValuesReaderState.DimTree dimTree = state.getDimTree(dim);
+ int dimOrd = dimTree.dimStartOrd;
+ return getPathResult(
+ dimConfig,
+ dim,
+ emptyPath,
+ dimOrd,
+ dimTree.iterator(),
+ topNChildren,
+ dimToChildOrdsResult);
+ } else {
+ OrdRange ordRange = state.getOrdRange(dim);
+ int dimOrd = ordRange.start;
+ PrimitiveIterator.OfInt childIt = ordRange.iterator();
+ if (dimConfig.multiValued && dimConfig.requireDimCount) {
+ // If the dim is multi-valued and requires dim counts, we know we've explicitly indexed
+ // the dimension and we need to skip past it so the iterator is positioned on the first
+ // child:
+ childIt.next();
+ }
+ return getPathResult(
+ dimConfig, dim, emptyPath, dimOrd, childIt, topNChildren, dimToChildOrdsResult);
+ }
+ }
+
@Override
public List<FacetResult> getAllDims(int topN) throws IOException {
validateTopN(topN);
List<FacetResult> results = new ArrayList<>();
for (String dim : state.getDims()) {
- FacetsConfig.DimConfig dimConfig = stateConfig.getDimConfig(dim);
- if (dimConfig.hierarchical) {
- SortedSetDocValuesReaderState.DimTree dimTree = state.getDimTree(dim);
- int dimOrd = dimTree.dimStartOrd;
- FacetResult fr = getPathResult(dimConfig, dim, emptyPath, dimOrd, dimTree.iterator(), topN);
- if (fr != null) {
- results.add(fr);
- }
- } else {
- OrdRange ordRange = state.getOrdRange(dim);
- int dimOrd = ordRange.start;
- PrimitiveIterator.OfInt childIt = ordRange.iterator();
- if (dimConfig.multiValued && dimConfig.requireDimCount) {
- // If the dim is multi-valued and requires dim counts, we know we've explicitly indexed
- // the dimension and we need to skip past it so the iterator is positioned on the first
- // child:
- childIt.next();
- }
- FacetResult fr = getPathResult(dimConfig, dim, emptyPath, dimOrd, childIt, topN);
- if (fr != null) {
- results.add(fr);
- }
+ FacetResult factResult = getFacetResultForDim(dim, topN);
+ if (factResult != null) {
+ results.add(factResult);
}
}
@@ -522,4 +618,112 @@ public class ConcurrentSortedSetDocValuesFacetCounts extends Facets {
return results;
}
+
+ @Override
+ public List<FacetResult> getTopDims(int topNDims, int topNChildren) throws IOException {
+ if (topNDims <= 0 || topNChildren <= 0) {
+ throw new IllegalArgumentException("topN must be > 0");
+ }
+
+ // Creates priority queue to store top dimensions and sort by their aggregated values/hits and
+ // string values.
+ PriorityQueue<DimValueResult> pq =
+ new PriorityQueue<>(topNDims) {
+ @Override
+ protected boolean lessThan(DimValueResult a, DimValueResult b) {
+ if (a.value > b.value) {
+ return false;
+ } else if (a.value < b.value) {
+ return true;
+ } else {
+ return a.dim.compareTo(b.dim) > 0;
+ }
+ }
+ };
+
+ HashMap<String, ChildOrdsResult> dimToChildOrdsResult = new HashMap<>();
+ int dimCount;
+
+ for (String dim : state.getDims()) {
+ FacetsConfig.DimConfig dimConfig = stateConfig.getDimConfig(dim);
+ if (dimConfig.hierarchical) {
+ SortedSetDocValuesReaderState.DimTree dimTree = state.getDimTree(dim);
+ int dimOrd = dimTree.dimStartOrd;
+ // get dim value
+ dimCount =
+ getDimValue(
+ dimConfig, dim, dimOrd, dimTree.iterator(), topNChildren, dimToChildOrdsResult);
+ } else {
+ OrdRange ordRange = state.getOrdRange(dim);
+ int dimOrd = ordRange.start;
+ PrimitiveIterator.OfInt childIt = ordRange.iterator();
+ if (dimConfig.multiValued && dimConfig.requireDimCount) {
+ // If the dim is multi-valued and requires dim counts, we know we've explicitly indexed
+ // the dimension and we need to skip past it so the iterator is positioned on the first
+ // child:
+ childIt.next();
+ }
+ dimCount = getDimValue(dimConfig, dim, dimOrd, childIt, topNChildren, dimToChildOrdsResult);
+ }
+
+ if (dimCount != 0) {
+ // use priority queue to store DimValueResult for topNDims
+ if (pq.size() < topNDims) {
+ pq.add(new DimValueResult(dim, dimCount));
+ } else {
+ if (dimCount > pq.top().value
+ || (dimCount == pq.top().value && dim.compareTo(pq.top().dim) < 0)) {
+ DimValueResult bottomDim = pq.top();
+ bottomDim.dim = dim;
+ bottomDim.value = dimCount;
+ pq.updateTop();
+ }
+ }
+ }
+ }
+
+ // get FacetResult for topNDims
+ int resultSize = pq.size();
+ FacetResult[] results = new FacetResult[resultSize];
+
+ while (pq.size() > 0) {
+ DimValueResult dimValueResult = pq.pop();
+ FacetResult facetResult =
+ getFacetResultForDim(
+ dimValueResult.dim, topNChildren, dimToChildOrdsResult.get(dimValueResult.dim));
+ resultSize--;
+ results[resultSize] = facetResult;
+ }
+ return Arrays.asList(results);
+ }
+
+ /**
+ * Creates ChildOrdsResult to store dimCount, childCount, and the queue for the dimension's top
+ * children
+ */
+ private static class ChildOrdsResult {
+ final int dimCount;
+ final int childCount;
+ final TopOrdAndIntQueue q;
+
+ ChildOrdsResult(int dimCount, int childCount, TopOrdAndIntQueue q) {
+ this.dimCount = dimCount;
+ this.childCount = childCount;
+ this.q = q;
+ }
+ }
+
+ /**
+ * Creates DimValueResult to store the label and value of dim in order to sort by these two
+ * fields.
+ */
+ private static class DimValueResult {
+ String dim;
+ int value;
+
+ DimValueResult(String dim, int value) {
+ this.dim = dim;
+ this.value = value;
+ }
+ }
}
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java b/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java
index 80499b3d0f8..8b68db01db4 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/sortedset/TestSortedSetDocValuesFacets.java
@@ -435,6 +435,19 @@ public class TestSortedSetDocValuesFacets extends FacetTestCase {
assertEquals(
"dim=a path=[] value=1 childCount=1\n bar (1)\n",
facets.getTopChildren(10, "a").toString());
+
+ // test getTopDims in ConcurrentSortedSetDocValuesFacetCounts
+ List<FacetResult> results = facets.getAllDims(10);
+ // test getTopDims(10, 10) and expect same results from getAllDims(10)
+ List<FacetResult> allTopDimsResults = facets.getTopDims(10, 10);
+
+ // test getTopDims(n, 10)
+ if (allTopDimsResults.size() > 0) {
+ for (int i = 1; i < results.size(); i++) {
+ assertEquals(results.subList(0, i), facets.getTopDims(i, 10));
+ }
+ }
+
} finally {
exec.shutdownNow();
}
@@ -495,6 +508,18 @@ public class TestSortedSetDocValuesFacets extends FacetTestCase {
assertEquals(
"dim=b path=[buzz] value=1 childCount=1\n baz (1)\n",
facets.getTopChildren(10, "b", "buzz").toString());
+
+ // test getTopDims in ConcurrentSortedSetDocValuesFacetCounts
+ List<FacetResult> results = facets.getAllDims(10);
+ // test getTopDims(10, 10) and expect same results from getAllDims(10)
+ List<FacetResult> allTopDimsResults = facets.getTopDims(10, 10);
+
+ // test getTopDims(n, 10)
+ if (allTopDimsResults.size() > 0) {
+ for (int i = 1; i < results.size(); i++) {
+ assertEquals(results.subList(0, i), facets.getTopDims(i, 10));
+ }
+ }
} finally {
exec.shutdownNow();
}