Apache Solr open-source search software
SOLR-17182: Part 1: add ExitableDirectoryReader benchmark. (#3739)
Andrzej Białecki29c96eb55db9
diff --git a/solr/benchmark/build.gradle b/solr/benchmark/build.gradleindex bf32ee89457..48c97747f04 100644--- a/solr/benchmark/build.gradle+++ b/solr/benchmark/build.gradle@@ -42,6 +42,7 @@ task echoCp {dependencies {implementation project(':solr:test-framework')+ implementation project(':solr:core')implementation project(':solr:solrj')implementation project(':solr:solrj-streaming')diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/ExitableDirectoryReaderSearch.java b/solr/benchmark/src/java/org/apache/solr/bench/search/ExitableDirectoryReaderSearch.javanew file mode 100644index 00000000000..dc5e3af9f4d--- /dev/null+++ b/solr/benchmark/src/java/org/apache/solr/bench/search/ExitableDirectoryReaderSearch.java@@ -0,0 +1,197 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.bench.search;++import static org.apache.solr.bench.BaseBenchState.log;+import static org.apache.solr.bench.generators.SourceDSL.integers;+import static org.apache.solr.bench.generators.SourceDSL.strings;++import java.util.Map;+import java.util.Set;+import java.util.concurrent.TimeUnit;+import org.apache.solr.bench.Docs;+import org.apache.solr.bench.MiniClusterState;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.request.QueryRequest;+import org.apache.solr.client.solrj.response.QueryResponse;+import org.apache.solr.common.SolrInputDocument;+import org.apache.solr.common.params.ModifiableSolrParams;+import org.apache.solr.search.CallerSpecificQueryLimit;+import org.apache.solr.search.SolrIndexSearcher;+import org.apache.solr.util.TestInjection;+import org.openjdk.jmh.annotations.Benchmark;+import org.openjdk.jmh.annotations.BenchmarkMode;+import org.openjdk.jmh.annotations.Fork;+import org.openjdk.jmh.annotations.Level;+import org.openjdk.jmh.annotations.Measurement;+import org.openjdk.jmh.annotations.Mode;+import org.openjdk.jmh.annotations.OutputTimeUnit;+import org.openjdk.jmh.annotations.Param;+import org.openjdk.jmh.annotations.Scope;+import org.openjdk.jmh.annotations.Setup;+import org.openjdk.jmh.annotations.State;+import org.openjdk.jmh.annotations.TearDown;+import org.openjdk.jmh.annotations.Threads;+import org.openjdk.jmh.annotations.Warmup;+import org.openjdk.jmh.infra.Blackhole;++@Fork(value = 1)+@BenchmarkMode(Mode.AverageTime)+@Warmup(time = 20, iterations = 2)+@OutputTimeUnit(TimeUnit.MILLISECONDS)+@Measurement(time = 30, iterations = 4)+@Threads(value = 1)+public class ExitableDirectoryReaderSearch {++ static final String COLLECTION = "c1";++ @State(Scope.Benchmark)+ public static class BenchState {++ Docs queryFields;++ int NUM_DOCS = 500_000;+ int WORDS = NUM_DOCS / 100;++ @Setup(Level.Trial)+ public void setupTrial(MiniClusterState.MiniClusterBenchState miniClusterState)+ throws Exception {+ miniClusterState.setUseHttp1(true);+ System.setProperty("documentCache.enabled", "false");+ System.setProperty("queryResultCache.enabled", "false");+ System.setProperty("filterCache.enabled", "false");+ System.setProperty("miniClusterBaseDir", "build/work/mini-cluster");+ // create a lot of small segments+ System.setProperty("segmentsPerTier", "200");+ System.setProperty("maxBufferedDocs", "100");++ miniClusterState.startMiniCluster(1);+ log("######### Creating index ...");+ miniClusterState.createCollection(COLLECTION, 1, 1);+ // create a lot of large-ish fields to scan positions+ Docs docs =+ Docs.docs(1234567890L)+ .field("id", integers().incrementing())+ .field("f1_ts", strings().alpha().maxCardinality(WORDS).ofLengthBetween(3, 10))+ .field(+ "f2_ts", strings().alpha().maxCardinality(WORDS).multi(50).ofLengthBetween(3, 10))+ .field(+ "f3_ts", strings().alpha().maxCardinality(WORDS).multi(50).ofLengthBetween(3, 10))+ .field(+ "f4_ts", strings().alpha().maxCardinality(WORDS).multi(50).ofLengthBetween(3, 10))+ .field(+ "f5_ts", strings().alpha().maxCardinality(WORDS).multi(50).ofLengthBetween(3, 10))+ .field(+ "f6_ts", strings().alpha().maxCardinality(WORDS).multi(50).ofLengthBetween(3, 10))+ .field(+ "f7_ts", strings().alpha().maxCardinality(WORDS).multi(50).ofLengthBetween(3, 10))+ .field(+ "f8_ts", strings().alpha().maxCardinality(WORDS).multi(50).ofLengthBetween(3, 10))+ .field(+ "f9_ts",+ strings().alpha().maxCardinality(WORDS).multi(50).ofLengthBetween(3, 10));+ miniClusterState.index(COLLECTION, docs, NUM_DOCS, true);+ miniClusterState.forceMerge(COLLECTION, 200);+ miniClusterState.dumpCoreInfo();+ }++ @Param({"false", "true"})+ boolean useEDR;++ // this adds significant processing time to the checking of query limits+ // both to verify that it's actually used and to illustrate the impact of limit checking+ @Param({"false", "true"})+ boolean verifyEDRInUse = true;++ private static final String matchExpression = "ExitableTermsEnum:-1";++ @Setup(Level.Iteration)+ public void setupQueries(MiniClusterState.MiniClusterBenchState state) throws Exception {+ System.setProperty(SolrIndexSearcher.EXITABLE_READER_PROPERTY, String.valueOf(useEDR));+ if (verifyEDRInUse) {+ TestInjection.queryTimeout = new CallerSpecificQueryLimit(Set.of(matchExpression));+ }+ // reload collection to force searcher / reader refresh+ CollectionAdminRequest.Reload reload = CollectionAdminRequest.reloadCollection(COLLECTION);+ state.client.request(reload);++ queryFields =+ Docs.docs(1234567890L)+ .field("id", integers().incrementing())+ .field("f1_ts", strings().alpha().maxCardinality(WORDS).ofLengthBetween(3, 10))+ .field(+ "f2_ts", strings().alpha().maxCardinality(WORDS).multi(5).ofLengthBetween(3, 10));+ }++ @TearDown(Level.Iteration)+ public void tearDownTrial() throws Exception {+ if (useEDR && verifyEDRInUse) {+ CallerSpecificQueryLimit queryLimit = (CallerSpecificQueryLimit) TestInjection.queryTimeout;+ if (queryLimit == null) {+ throw new RuntimeException("Missing setup!");+ }+ Map<String, Integer> callCounts = queryLimit.getCallerMatcher().getCallCounts();+ log("######### Caller specific stats:");+ log("Call counts: " + callCounts);+ if (callCounts.get(matchExpression) == null) {+ throw new RuntimeException("Missing call counts!");+ }+ if (callCounts.get(matchExpression).intValue() == 0) {+ throw new RuntimeException("No call counts!");+ }+ }+ }+ }++ private static ModifiableSolrParams createInitialParams() {+ ModifiableSolrParams params =+ MiniClusterState.params("rows", "100", "timeAllowed", "1000", "fl", "*");+ return params;+ }++ @Benchmark+ public void testShortQuery(+ MiniClusterState.MiniClusterBenchState miniClusterState, Blackhole bh, BenchState state)+ throws Exception {+ SolrInputDocument queryDoc = state.queryFields.inputDocument();+ ModifiableSolrParams params = createInitialParams();+ params.set("q", "f1_ts:" + queryDoc.getFieldValue("f1_ts").toString());+ QueryRequest queryRequest = new QueryRequest(params);+ QueryResponse rsp = queryRequest.process(miniClusterState.client, COLLECTION);+ bh.consume(rsp);+ }++ @Benchmark+ public void testLongQuery(+ MiniClusterState.MiniClusterBenchState miniClusterState, Blackhole bh, BenchState state)+ throws Exception {+ SolrInputDocument queryDoc = state.queryFields.inputDocument();+ ModifiableSolrParams params = createInitialParams();+ StringBuilder query = new StringBuilder();+ for (int i = 2; i < 10; i++) {+ if (query.length() > 0) {+ query.append(" ");+ }+ String fld = "f" + i + "_ts";+ query.append(fld + ":\"" + queryDoc.getFieldValue(fld) + "\"~20");+ }+ params.set("q", query.toString());+ QueryRequest queryRequest = new QueryRequest(params);+ QueryResponse rsp = queryRequest.process(miniClusterState.client, COLLECTION);+ bh.consume(rsp);+ }+}diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/JsonFaceting.java b/solr/benchmark/src/java/org/apache/solr/bench/search/JsonFaceting.javaindex 1595de56d3b..d83477b56f7 100755--- a/solr/benchmark/src/java/org/apache/solr/bench/search/JsonFaceting.java+++ b/solr/benchmark/src/java/org/apache/solr/bench/search/JsonFaceting.java@@ -28,6 +28,7 @@ import org.apache.solr.bench.MiniClusterState;import org.apache.solr.client.solrj.request.QueryRequest;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.util.NamedList;+import org.apache.solr.search.SolrIndexSearcher;import org.openjdk.jmh.annotations.Benchmark;import org.openjdk.jmh.annotations.BenchmarkMode;import org.openjdk.jmh.annotations.Fork;@@ -43,6 +44,7 @@ import org.openjdk.jmh.annotations.Threads;import org.openjdk.jmh.annotations.Timeout;import org.openjdk.jmh.annotations.Warmup;import org.openjdk.jmh.infra.BenchmarkParams;+import org.openjdk.jmh.infra.Blackhole;/** A benchmark to experiment with the performance of json faceting. */@BenchmarkMode(Mode.Throughput)@@ -71,6 +73,12 @@ public class JsonFaceting {@Param("4")int numShards;+ @Param({"false", "true"})+ boolean useTimeLimit;++ @Param({"false", "true"})+ boolean useExitableDirectoryReader;+// DV, // DocValues, collect into ordinal array// UIF, // UnInvertedField, collect into ordinal array// DVHASH, // DocValues, collect into hash@@ -99,8 +107,13 @@ public class JsonFaceting {BenchmarkParams benchmarkParams, MiniClusterState.MiniClusterBenchState miniClusterState)throws Exception {- System.setProperty("maxMergeAtOnce", "30");- System.setProperty("segmentsPerTier", "30");+ System.setProperty("maxMergeAtOnce", "50");+ System.setProperty("segmentsPerTier", "50");+ if (useExitableDirectoryReader) {+ System.setProperty(SolrIndexSearcher.EXITABLE_READER_PROPERTY, "true");+ } else {+ System.setProperty(SolrIndexSearcher.EXITABLE_READER_PROPERTY, "false");+ }miniClusterState.startMiniCluster(nodeCount);@@ -158,6 +171,11 @@ public class JsonFaceting {+ " , f8:{type:terms, field:'facet_s', limit:2, sort:'x desc', facet:{x:'countvals(int4_i_dv)'} } "+ '}');+ if (useTimeLimit) {+ // high enough to return all results, but still affecting the performance+ params.set("timeAllowed", "5000");+ }+// MiniClusterState.log("params: " + params + "\n");}@@ -175,10 +193,11 @@ public class JsonFaceting {@Benchmark@Timeout(time = 500, timeUnit = TimeUnit.SECONDS)- public Object jsonFacet(+ public void jsonFacet(MiniClusterState.MiniClusterBenchState miniClusterState,BenchState state,- BenchState.ThreadState threadState)+ BenchState.ThreadState threadState,+ Blackhole bh)throws Exception {final var url = miniClusterState.nodes.get(threadState.random.nextInt(state.nodeCount));QueryRequest queryRequest = new QueryRequest(state.params);@@ -190,6 +209,6 @@ public class JsonFaceting {// MiniClusterState.log("result: " + result);- return result;+ bh.consume(result);}}diff --git a/solr/core/src/test/org/apache/solr/core/ExitableDirectoryReaderTest.java b/solr/core/src/test/org/apache/solr/core/ExitableDirectoryReaderTest.javaindex 86e4dcc51a0..61f24092280 100644--- a/solr/core/src/test/org/apache/solr/core/ExitableDirectoryReaderTest.java+++ b/solr/core/src/test/org/apache/solr/core/ExitableDirectoryReaderTest.java@@ -17,6 +17,7 @@package org.apache.solr.core;import java.util.Map;+import java.util.Set;import org.apache.solr.SolrTestCaseJ4;import org.apache.solr.search.CallerSpecificQueryLimit;import org.apache.solr.search.SolrIndexSearcher;@@ -69,11 +70,11 @@ public class ExitableDirectoryReaderTest extends SolrTestCaseJ4 {// create a limit that will not trip but will report calls to shouldExit()// NOTE: we need to use the inner class name to capture the callsString callerExpr = "ExitableTermsEnum:-1";- CallerSpecificQueryLimit queryLimit = new CallerSpecificQueryLimit(callerExpr);+ CallerSpecificQueryLimit queryLimit = new CallerSpecificQueryLimit(Set.of(callerExpr));TestInjection.queryTimeout = queryLimit;String q = "name:a*";assertJQ(req("q", q), assertionString);- Map<String, Integer> callCounts = queryLimit.getCallCounts();+ Map<String, Integer> callCounts = queryLimit.getCallerMatcher().getCallCounts();if (withExitableDirectoryReader) {assertTrue("there should be some calls from ExitableTermsEnum: " + callCounts,@@ -86,7 +87,7 @@ public class ExitableDirectoryReaderTest extends SolrTestCaseJ4 {// check that the limits are tripped in ExitableDirectoryReader if it's in useint maxCount = random().nextInt(10) + 1;callerExpr = "ExitableTermsEnum:" + maxCount;- queryLimit = new CallerSpecificQueryLimit(callerExpr);+ queryLimit = new CallerSpecificQueryLimit(Set.of(callerExpr));TestInjection.queryTimeout = queryLimit;// avoid using the cacheq = "name:b*";@@ -95,7 +96,7 @@ public class ExitableDirectoryReaderTest extends SolrTestCaseJ4 {} else {assertJQ(req("q", q), assertionString);}- callCounts = queryLimit.getCallCounts();+ callCounts = queryLimit.getCallerMatcher().getCallCounts();if (withExitableDirectoryReader) {assertTrue("there should be at least " + maxCount + " calls from ExitableTermsEnum: " + callCounts,diff --git a/solr/core/src/test/org/apache/solr/search/TestQueryLimits.java b/solr/core/src/test/org/apache/solr/search/TestQueryLimits.javaindex 690b05e3f21..dbee1511295 100644--- a/solr/core/src/test/org/apache/solr/search/TestQueryLimits.java+++ b/solr/core/src/test/org/apache/solr/search/TestQueryLimits.java@@ -17,6 +17,7 @@package org.apache.solr.search;import java.util.Map;+import java.util.Set;import org.apache.lucene.tests.util.TestUtil;import org.apache.solr.client.solrj.SolrClient;import org.apache.solr.client.solrj.request.CollectionAdminRequest;@@ -79,7 +80,7 @@ public class TestQueryLimits extends SolrCloudTestCase {"FacetComponent.process:2"};for (String matchingExpr : matchingExprTests) {- CallerSpecificQueryLimit limit = new CallerSpecificQueryLimit(matchingExpr);+ CallerSpecificQueryLimit limit = new CallerSpecificQueryLimit(Set.of(matchingExpr));TestInjection.queryTimeout = limit;rsp =solrClient.query(@@ -98,11 +99,14 @@ public class TestQueryLimits extends SolrCloudTestCase {assertNotNull("should have partial results for expr " + matchingExpr,rsp.getHeader().get("partialResults"));- assertFalse("should have trippedBy info", limit.getTrippedBy().isEmpty());+ assertFalse("should have trippedBy info", limit.getCallerMatcher().getTrippedBy().isEmpty());assertTrue(- "expected result to start with " + matchingExpr + " but was " + limit.getTrippedBy(),- limit.getTrippedBy().iterator().next().startsWith(matchingExpr));- Map<String, Integer> callCounts = limit.getCallCounts();+ "expected result to start with "+ + matchingExpr+ + " but was "+ + limit.getCallerMatcher().getTrippedBy(),+ limit.getCallerMatcher().getTrippedBy().iterator().next().startsWith(matchingExpr));+ Map<String, Integer> callCounts = limit.getCallerMatcher().getCallCounts();assertTrue("call count should be > 0", callCounts.get(matchingExpr) > 0);}}diff --git a/solr/test-framework/src/java/org/apache/solr/search/CallerSpecificQueryLimit.java b/solr/test-framework/src/java/org/apache/solr/search/CallerSpecificQueryLimit.javaindex 0deed2273e9..559a798296b 100644--- a/solr/test-framework/src/java/org/apache/solr/search/CallerSpecificQueryLimit.java+++ b/solr/test-framework/src/java/org/apache/solr/search/CallerSpecificQueryLimit.java@@ -17,193 +17,45 @@package org.apache.solr.search;import java.lang.invoke.MethodHandles;-import java.util.Arrays;import java.util.Collection;-import java.util.HashMap;-import java.util.HashSet;-import java.util.List;-import java.util.Map;-import java.util.Optional;import java.util.Set;-import java.util.concurrent.ConcurrentHashMap;-import java.util.concurrent.atomic.AtomicInteger;-import java.util.stream.Collectors;+import org.apache.solr.util.CallerMatcher;import org.slf4j.Logger;import org.slf4j.LoggerFactory;/*** Helper class to simulate query timeouts at specific points in various components that call {@link- * QueryLimits#shouldExit()}. These calling points are identified by the calling class' simple name- * and optionally a method name and the optional maximum count, e.g. <code>MoreLikeThisComponent- * </code> or <code>- * ClusteringComponent.finishStage</code>, <code>ClusteringComponent.finishStage:100</code>.- *- * <p>NOTE: implementation details cause the expression <code>simpleName</code> to be disabled when- * also any <code>simpleName.anyMethod[:NNN]</code> expression is used for the same class name.- *- * <p>NOTE 2: when maximum count is a negative number e.g. <code>simpleName.someMethod:-1</code>- * then only the number of calls to {@link QueryLimits#shouldExit()} for that expression will be- * reported but no limit will be enforced.+ * QueryLimits#shouldExit()}. This class uses {@link CallerMatcher} to collect the matching callers+ * information and enforce the count limits.*/public class CallerSpecificQueryLimit implements QueryLimit {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());- private final StackWalker stackWalker =- StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);- // className -> set of method names- private final Map<String, Set<String>> interestingCallers = new HashMap<>();- // expr -> initial count- private final Map<String, Integer> maxCounts = new ConcurrentHashMap<>();- // expr -> current count- private final Map<String, AtomicInteger> callCounts = new ConcurrentHashMap<>();- private Set<String> trippedBy = ConcurrentHashMap.newKeySet();+ private final CallerMatcher callerMatcher;/*** Signal a timeout in places that match the calling classes (and methods).** @param callerExprs list of expressions in the format of <code>- * simpleClassName[.methodName][:NNN]</code>. If the list is empty or null then the first call- * to {@link #shouldExit()} from any caller will match.+ * ( simpleClassName[.methodName] | * )[:NNN]</code>. If the list is empty or null then the+ * first call to {@link #shouldExit()} from any caller will match.*/- public CallerSpecificQueryLimit(String... callerExprs) {- this(callerExprs != null ? Arrays.asList(callerExprs) : List.of());- }-public CallerSpecificQueryLimit(Collection<String> callerExprs) {- for (String callerExpr : callerExprs) {- String[] exprCount = callerExpr.split(":");- if (exprCount.length > 2) {- throw new RuntimeException("Invalid count in callerExpr: " + callerExpr);- }- String[] clazzMethod = exprCount[0].split("\\.");- if (clazzMethod.length > 2) {- throw new RuntimeException("Invalid method in callerExpr: " + callerExpr);- }- Set<String> methods =- interestingCallers.computeIfAbsent(clazzMethod[0], c -> new HashSet<>());- if (clazzMethod.length > 1) {- methods.add(clazzMethod[1]);- }- if (exprCount.length > 1) {- try {- int count = Integer.parseInt(exprCount[1]);- maxCounts.put(exprCount[0], count);- callCounts.put(exprCount[0], new AtomicInteger(0));- } catch (NumberFormatException e) {- throw new RuntimeException("Invalid count in callerExpr: " + callerExpr, e);- }- }- }- }-- /** Returns the set of caller expressions that were tripped. */- public Set<String> getTrippedBy() {- return trippedBy;+ // exclude myself and QueryLimits+ callerMatcher =+ new CallerMatcher(+ callerExprs,+ Set.of(+ CallerSpecificQueryLimit.class.getSimpleName(), QueryLimits.class.getSimpleName()));}- /** Returns a map of tripped caller expressions to their current call counts. */- public Map<String, Integer> getCallCounts() {- return callCounts.entrySet().stream()- .collect(- Collectors.toMap(- e ->- e.getKey()- + (maxCounts.containsKey(e.getKey())- ? ":" + maxCounts.get(e.getKey())- : ""),- e -> e.getValue().get()));+ public CallerMatcher getCallerMatcher() {+ return callerMatcher;}@Overridepublic boolean shouldExit() {- Optional<String> matchingExpr =- stackWalker.walk(- s ->- s.filter(- frame -> {- Class<?> declaring = frame.getDeclaringClass();- // skip bottom-most frames: myself and QueryLimits- if (declaring == this.getClass() || declaring == QueryLimits.class) {- return false;- }- String method = frame.getMethodName();- if (interestingCallers.isEmpty()) {- // any caller is an offending caller- String expr = declaring.getSimpleName() + "." + method;- if (log.isInfoEnabled()) {- log.info("++++ Limit tripped by any first caller: {} ++++", expr);- }- trippedBy.add(expr);- callCounts- .computeIfAbsent(expr, k -> new AtomicInteger())- .incrementAndGet();- return true;- }- Set<String> methods = interestingCallers.get(declaring.getSimpleName());- if (methods == null) {- // no class and no methods specified for this class, so skip- return false;- }- // MATCH. Class name was specified, possibly with methods.- // If methods is empty then all methods match, otherwise only the- // specified methods match.- if (methods.isEmpty() || methods.contains(method)) {- String expr = declaring.getSimpleName();- if (methods.contains(method)) {- expr = expr + "." + method;- } else {- // even though we don't match/enforce at the method level, still- // record the method counts to give better insight into the callers- callCounts- .computeIfAbsent(- declaring.getSimpleName() + "." + method,- k -> new AtomicInteger(0))- .incrementAndGet();- }- int currentCount =- callCounts- .computeIfAbsent(expr, k -> new AtomicInteger(0))- .incrementAndGet();- // check if we have a max count for this expression- if (maxCounts.containsKey(expr)) {- int maxCount = maxCounts.getOrDefault(expr, 0);- // if max count is negative then just report the call count- if (maxCount < 0) {- maxCount = Integer.MAX_VALUE;- }- if (currentCount > maxCount) {- if (log.isInfoEnabled()) {- log.info(- "++++ Limit tripped by caller: {}, current count: {}, max: {} ++++",- expr,- currentCount,- maxCounts.get(expr));- }- trippedBy.add(expr + ":" + maxCounts.get(expr));- return true;- } else {- return false; // max count not reached, not tripped yet- }- } else {- trippedBy.add(expr);- if (log.isInfoEnabled()) {- log.info("++++ Limit tripped by caller: {} ++++", expr);- }- return true; // no max count, so tripped on first call- }- } else {- return false;- }- })- .map(- frame ->- (frame.getDeclaringClass().getSimpleName().isBlank()- ? frame.getClassName()- : frame.getDeclaringClass().getSimpleName())- + "."- + frame.getMethodName())- .findFirst());- return matchingExpr.isPresent();+ return callerMatcher.checkCaller().isPresent();}@Overridediff --git a/solr/test-framework/src/java/org/apache/solr/util/CallerMatcher.java b/solr/test-framework/src/java/org/apache/solr/util/CallerMatcher.javanew file mode 100644index 00000000000..22030d357e3--- /dev/null+++ b/solr/test-framework/src/java/org/apache/solr/util/CallerMatcher.java@@ -0,0 +1,270 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.util;++import java.lang.invoke.MethodHandles;+import java.util.Collection;+import java.util.Collections;+import java.util.HashMap;+import java.util.HashSet;+import java.util.Map;+import java.util.Optional;+import java.util.Set;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.atomic.AtomicInteger;+import java.util.stream.Collectors;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++/**+ * Helper class to collect interesting callers at specific points. These calling points are+ * identified by the calling class' simple name and optionally a method name and the optional+ * maximum count, e.g. <code>MoreLikeThisComponent</code> or <code>ClusteringComponent.finishStage+ * </code>, <code>ClusteringComponent.finishStage:100</code>. A single wildcard name <code>*</code>+ * may be used to mean "any class", which may be useful to e.g. collect all callers using an+ * expression <code>*:-1</code>.+ *+ * <p>Within your caller you should invoke {@link #checkCaller()} to count any matching frames in+ * the current stack, and check if any of the count limits has been reached. Each invocation will+ * increase the call count of the matching expression(s). For one invocation multiple matching+ * expression counts can be affected because all current stack frames are examined against the+ * matching expressions.+ *+ * <p>NOTE: implementation details cause the expression <code>simpleName[:NNN]</code> to be disabled+ * when also any <code>simpleName.anyMethod[:NNN]</code> expression is used for the same class name.+ *+ * <p>NOTE 2: when maximum count is a negative number e.g. <code>simpleName[.someMethod]:-1</code>+ * then only the number of calls to {@link #checkCaller()} for the matching expressions will be+ * reported but {@link #checkCaller()} will never return this expression.+ */+public class CallerMatcher {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ public static final String WILDCARD = "*";++ private final StackWalker stackWalker =+ StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE);+ // className -> set of method names+ private final Map<String, Set<String>> exactCallers = new HashMap<>();+ private final Map<String, Set<String>> excludeCallers = new HashMap<>();+ // expr -> initial count+ private final Map<String, Integer> maxCounts = new ConcurrentHashMap<>();+ // expr -> current count+ private final Map<String, AtomicInteger> callCounts = new ConcurrentHashMap<>();+ private Set<String> trippedBy = ConcurrentHashMap.newKeySet();++ /**+ * Create an instance that reacts to the specified caller expressions.+ *+ * @param callerExprs list of expressions in the format of <code>+ * ( simpleClassName[.methodName] | * )[:NNN]</code>. If the list is empty or null then the+ * first call to {@link #checkCaller()} ()} from any caller will match.+ */+ public CallerMatcher(Collection<String> callerExprs, Collection<String> excludeExprs) {+ for (String callerExpr : callerExprs) {+ String[] exprCount = callerExpr.split(":");+ if (exprCount.length > 2) {+ throw new RuntimeException("Invalid count in callerExpr: " + callerExpr);+ }+ String[] clazzMethod = exprCount[0].split("\\.");+ if (clazzMethod.length > 2) {+ throw new RuntimeException("Invalid method in callerExpr: " + callerExpr);+ }+ Set<String> methods = exactCallers.computeIfAbsent(clazzMethod[0], c -> new HashSet<>());+ if (clazzMethod.length > 1) {+ methods.add(clazzMethod[1]);+ }+ if (exprCount.length > 1) {+ try {+ int count = Integer.parseInt(exprCount[1]);+ maxCounts.put(exprCount[0], count);+ callCounts.put(exprCount[0], new AtomicInteger(0));+ } catch (NumberFormatException e) {+ throw new RuntimeException("Invalid count in callerExpr: " + callerExpr, e);+ }+ }+ }+ for (String excludeExpr : excludeExprs) {+ String[] clazzMethod = excludeExpr.split("\\.");+ if (clazzMethod.length > 2) {+ throw new RuntimeException("Invalid method in excludeExpr: " + excludeExpr);+ }+ Set<String> methods = excludeCallers.computeIfAbsent(clazzMethod[0], c -> new HashSet<>());+ if (clazzMethod.length > 1) {+ methods.add(clazzMethod[1]);+ }+ }+ }++ /**+ * Returns the set of caller expressions that were tripped (reached their count limit). This+ * method can be called after {@link #checkCaller()} returns a matching expression to obtain all+ * expressions that exceeded their count limits.+ */+ public Set<String> getTrippedBy() {+ return Collections.unmodifiableSet(trippedBy);+ }++ /** Returns a map of matched caller expressions to their current call counts. */+ public Map<String, Integer> getCallCounts() {+ return callCounts.entrySet().stream()+ .collect(+ Collectors.toMap(+ e ->+ e.getKey()+ + (maxCounts.containsKey(e.getKey())+ ? ":" + maxCounts.get(e.getKey())+ : ""),+ e -> e.getValue().get()));+ }++ /**+ * Returns the matching caller expression when its count limit was reached, or empty if no caller+ * or no count limit was reached. Each invocation increases the call count of the matching caller,+ * if any. It's up to the caller to decide whether to continue processing after this count limit+ * is reached. The matching expression returned by this call will be also present in {@link+ * #getTrippedBy()}.+ */+ public Optional<String> checkCaller() {+ return stackWalker.walk(+ s ->+ s.filter(+ frame -> {+ // handle exclusions first+ if (isExcluded(frame)) {+ return false;+ }++ String className = frame.getDeclaringClass().getSimpleName();+ String method = frame.getMethodName();+ // now handle the matching expressions++ if (processAnyMatching(className, method)) {+ return true;+ }++ Set<String> methods = exactCallers.get(className);+ boolean wildcardMatch = false;+ if (methods == null) {+ // check for wildcard+ methods = exactCallers.get(WILDCARD);+ if (methods == null) {+ // no class and no methods specified for this class, so skip+ return false;+ } else {+ wildcardMatch = true;+ }+ }+ // MATCH. Class name was specified, possibly with methods.+ // If methods is empty then all methods match, otherwise only the+ // specified methods match.+ if (methods.isEmpty() || methods.contains(method)) {+ String expr = wildcardMatch ? WILDCARD : className;+ if (methods.contains(method)) {+ expr = expr + "." + method;+ } else {+ // even though we don't match/enforce at the method level, still+ // record the method counts to give better insight into the callers+ callCounts+ .computeIfAbsent(className + "." + method, k -> new AtomicInteger(0))+ .incrementAndGet();+ }+ int currentCount =+ callCounts+ .computeIfAbsent(expr, k -> new AtomicInteger(0))+ .incrementAndGet();+ return processMatchWithLimit(expr, currentCount);+ } else {+ return false;+ }+ })+ .map(+ frame ->+ (frame.getDeclaringClass().getSimpleName().isBlank()+ ? frame.getClassName()+ : frame.getDeclaringClass().getSimpleName())+ + "."+ + frame.getMethodName())+ .findFirst());+ }++ private boolean isExcluded(StackWalker.StackFrame frame) {+ Class<?> declaring = frame.getDeclaringClass();+ String method = frame.getMethodName();+ // always skip myself+ if (declaring == this.getClass()) {+ return true;+ }+ // skip exclusions, if any+ Set<String> excludeMethods = excludeCallers.get(declaring.getSimpleName());+ if (excludeMethods != null) {+ // skip any method+ if (excludeMethods.isEmpty()) {+ return true;+ } else {+ // or only the matching method+ return excludeMethods.contains(method);+ }+ }+ return false;+ }++ private boolean processAnyMatching(String className, String method) {+ if (exactCallers.isEmpty()) {+ // any caller is an offending caller+ String expr = className + "." + method;+ if (log.isInfoEnabled()) {+ log.info("++++ Tripped by any first caller: {} ++++", expr);+ }+ trippedBy.add(expr);+ callCounts.computeIfAbsent(expr, k -> new AtomicInteger()).incrementAndGet();+ return true;+ } else {+ return false;+ }+ }++ private boolean processMatchWithLimit(String expr, int currentCount) {+ // check if we have a max count for this expression+ if (maxCounts.containsKey(expr)) {+ int maxCount = maxCounts.get(expr);+ // if max count is negative then just report the call count+ if (maxCount < 0) {+ maxCount = Integer.MAX_VALUE;+ }+ if (currentCount > maxCount) {+ if (log.isInfoEnabled()) {+ log.info(+ "++++ Tripped by caller: {}, current count: {}, max: {} ++++",+ expr,+ currentCount,+ maxCounts.get(expr));+ }+ trippedBy.add(expr + ":" + maxCounts.get(expr));+ return true;+ } else {+ return false; // max count not reached, not tripped yet+ }+ } else {+ trippedBy.add(expr);+ if (log.isInfoEnabled()) {+ log.info("++++ Tripped by caller: {} ++++", expr);+ }+ return true; // no max count, so tripped on first call+ }+ }+}diff --git a/solr/test-framework/src/test/org/apache/solr/search/CallerSpecificQueryLimitTest.java b/solr/test-framework/src/test/org/apache/solr/search/CallerSpecificQueryLimitTest.javaindex da8f58c73cf..5259d3d7480 100644--- a/solr/test-framework/src/test/org/apache/solr/search/CallerSpecificQueryLimitTest.java+++ b/solr/test-framework/src/test/org/apache/solr/search/CallerSpecificQueryLimitTest.java@@ -115,14 +115,14 @@ public class CallerSpecificQueryLimitTest extends SolrTestCaseJ4 {matchingCallCounts.add(matchingClassName + ".doWork");}- CallerSpecificQueryLimit limit = new CallerSpecificQueryLimit(callerExpr);+ CallerSpecificQueryLimit limit = new CallerSpecificQueryLimit(Set.of(callerExpr));LimitedWorker limitedWorker = new LimitedWorker(limit);LimitedWorker2 limitedWorker2 = new LimitedWorker2(limit);for (int i = 0; i < count * 2; i++) {limitedWorker2.doWork();limitedWorker.doWork();}- Set<String> trippedBy = limit.getTrippedBy();+ Set<String> trippedBy = limit.getCallerMatcher().getTrippedBy();if (shouldTrip) {assertFalse("Limit should have been tripped, callerExpr: " + callerExpr, trippedBy.isEmpty());for (String nonMatchingCallerExpr : nonMatchingCallerExprs) {@@ -141,7 +141,7 @@ public class CallerSpecificQueryLimitTest extends SolrTestCaseJ4 {+ trippedBy,trippedBy.isEmpty());}- Map<String, Integer> callCounts = limit.getCallCounts();+ Map<String, Integer> callCounts = limit.getCallerMatcher().getCallCounts();for (String matchingCallCount : matchingCallCounts) {assertTrue("Call count for " + matchingCallCount + " should be > 0, callCounts: " + callCounts,