Apache Solr open-source search software
SOLR-17099: snitch does not return spurious tags (#2278)
Pierre Salagnacd1a4cbe487a0
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 79a4bb6e63f..bbac86c4c16 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -130,6 +130,8 @@ Optimizations* SOLR-17257: Both Minimize Cores and the Affinity replica placement strategies would over-gathercluster metrics. AttributeFetcherImpl was improved accordingly. (Yohann Callea)+* SOLR-17099: Do not return spurious tags when fetching metrics from a Solr node to another. (Pierre Salagnac)+Bug Fixes---------------------* SOLR-12813: subqueries should respect basic auth. (Rudy Seitz via Eric Pugh)@@ -230,7 +232,6 @@ Optimizations* GITHUB#2217: Scale to 10K+ collections better in ZkStateReader.refreshCollectionsList (David Smiley)-Bug Fixes---------------------* SOLR-17153: CloudSolrClient could fail a request immediately following a collection creation. Required double-checking the collection doesn’t exist. (Aparna Suresh via David Smiley)diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/NodeValueFetcher.java b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/NodeValueFetcher.javaindex c6e122aae50..56ec552723d 100644--- a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/NodeValueFetcher.java+++ b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/NodeValueFetcher.java@@ -18,6 +18,7 @@package org.apache.solr.client.solrj.impl;import java.util.ArrayList;+import java.util.EnumSet;import java.util.HashMap;import java.util.HashSet;import java.util.List;@@ -102,11 +103,12 @@ public class NodeValueFetcher {}}- private void getRemoteInfo(- String solrNode, Set<String> requestedTags, SolrClientNodeStateProvider.RemoteCallCtx ctx) {- if (!(ctx).isNodeAlive(solrNode)) return;+ /** Retrieve values that match JVM system properties and metrics. */+ private void getRemotePropertiesAndMetrics(+ Set<String> requestedTagNames, SolrClientNodeStateProvider.RemoteCallCtx ctx) {+Map<String, Set<Object>> metricsKeyVsTag = new HashMap<>();- for (String tag : requestedTags) {+ for (String tag : requestedTagNames) {if (tag.startsWith(SYSPROP)) {metricsKeyVsTag.computeIfAbsent(@@ -120,28 +122,42 @@ public class NodeValueFetcher {}}if (!metricsKeyVsTag.isEmpty()) {- SolrClientNodeStateProvider.fetchReplicaMetrics(solrNode, ctx, metricsKeyVsTag);+ SolrClientNodeStateProvider.fetchReplicaMetrics(ctx.getNode(), ctx, metricsKeyVsTag);}+ }- Set<String> groups = new HashSet<>();- List<String> prefixes = new ArrayList<>();+ /** Retrieve values of well known tags, as defined in {@link Tags}. */+ private void getRemoteTags(+ Set<String> requestedTagNames, SolrClientNodeStateProvider.RemoteCallCtx ctx) {++ // First resolve names into actual Tags instances+ EnumSet<Tags> requestedTags = EnumSet.noneOf(Tags.class);for (Tags t : Tags.values()) {- if (requestedTags.contains(t.tagName)) {- groups.add(t.group);- prefixes.add(t.prefix);+ if (requestedTagNames.contains(t.tagName)) {+ requestedTags.add(t);}}- if (groups.isEmpty() || prefixes.isEmpty()) return;+ if (requestedTags.isEmpty()) {+ return;+ }++ Set<String> groups = new HashSet<>();+ List<String> prefixes = new ArrayList<>();+ for (Tags t : requestedTags) {+ groups.add(t.group);+ prefixes.add(t.prefix);+ }ModifiableSolrParams params = new ModifiableSolrParams();params.add("group", StrUtils.join(groups, ','));params.add("prefix", StrUtils.join(prefixes, ','));try {- SimpleSolrResponse rsp = ctx.invokeWithRetry(solrNode, CommonParams.METRICS_PATH, params);+ SimpleSolrResponse rsp =+ ctx.invokeWithRetry(ctx.getNode(), CommonParams.METRICS_PATH, params);NamedList<?> metrics = (NamedList<?>) rsp.getResponse().get("metrics");if (metrics != null) {- for (Tags t : Tags.values()) {+ for (Tags t : requestedTags) {ctx.tags.put(t.tagName, t.extractResult(metrics));}}@@ -150,19 +166,26 @@ public class NodeValueFetcher {}}- public void getTags(- String solrNode, Set<String> requestedTags, SolrClientNodeStateProvider.RemoteCallCtx ctx) {+ public void getTags(Set<String> requestedTags, SolrClientNodeStateProvider.RemoteCallCtx ctx) {+try {- if (requestedTags.contains(NODE)) ctx.tags.put(NODE, solrNode);+ if (requestedTags.contains(NODE)) ctx.tags.put(NODE, ctx.getNode());if (requestedTags.contains(HOST)) {- Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);+ Matcher hostAndPortMatcher = hostAndPortPattern.matcher(ctx.getNode());if (hostAndPortMatcher.find()) ctx.tags.put(HOST, hostAndPortMatcher.group(1));}if (requestedTags.contains(PORT)) {- Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);+ Matcher hostAndPortMatcher = hostAndPortPattern.matcher(ctx.getNode());if (hostAndPortMatcher.find()) ctx.tags.put(PORT, hostAndPortMatcher.group(2));}- getRemoteInfo(solrNode, requestedTags, ctx);++ if (!ctx.isNodeAlive(ctx.getNode())) {+ // Don't try to reach out to the node if we already know it is down+ return;+ }++ getRemotePropertiesAndMetrics(requestedTags, ctx);+ getRemoteTags(requestedTags, ctx);} catch (Exception e) {throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);}diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.javaindex 4d7f087426a..7e70378c582 100644--- a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java+++ b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java@@ -115,7 +115,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriterprotected Map<String, Object> fetchTagValues(String node, Collection<String> tags) {NodeValueFetcher nodeValueFetcher = new NodeValueFetcher();RemoteCallCtx ctx = new RemoteCallCtx(node, solrClient);- nodeValueFetcher.getTags(node, new HashSet<>(tags), ctx);+ nodeValueFetcher.getTags(new HashSet<>(tags), ctx);return ctx.tags;}@@ -228,7 +228,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriterZkClientClusterStateProvider zkClientClusterStateProvider;CloudLegacySolrClient solrClient;public final Map<String, Object> tags = new HashMap<>();- private String node;+ private final String node;public Map<String, Object> session;public boolean isNodeAlive(String node) {diff --git a/solr/solrj-zookeeper/src/test/org/apache/solr/client/solrj/impl/NodeValueFetcherTest.java b/solr/solrj-zookeeper/src/test/org/apache/solr/client/solrj/impl/NodeValueFetcherTest.javanew file mode 100644index 00000000000..5f521dbb20c--- /dev/null+++ b/solr/solrj-zookeeper/src/test/org/apache/solr/client/solrj/impl/NodeValueFetcherTest.java@@ -0,0 +1,89 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.client.solrj.impl;++import java.util.Set;+import org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider.RemoteCallCtx;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.response.CollectionAdminResponse;+import org.apache.solr.cloud.SolrCloudTestCase;+import org.apache.solr.embedded.JettySolrRunner;+import org.junit.After;+import org.junit.Before;+import org.junit.BeforeClass;+import org.junit.Test;++public class NodeValueFetcherTest extends SolrCloudTestCase {++ private static final String COLLECTION =+ NodeValueFetcherTest.class.getSimpleName() + "_collection";++ private static int numShards;+ private static int numReplicas;++ @BeforeClass+ public static void setupCluster() throws Exception {+ // Metrics should be enabled to get them with snitch+ System.setProperty("metricsEnabled", "true");++ numShards = random().nextInt(3) + 3;+ numReplicas = random().nextInt(2) + 2;+ int numNodes = random().nextInt(2) + 1;++ configureCluster(numNodes).configure();+ }++ @Before+ public void setupCollection() throws Exception {+ CollectionAdminResponse rsp =+ CollectionAdminRequest.createCollection(COLLECTION, numShards, numReplicas)+ .process(cluster.getSolrClient());+ assertTrue(rsp.isSuccess());+ cluster.waitForActiveCollection(COLLECTION, numShards, numShards * numReplicas);+ }++ @After+ public void cleanup() throws Exception {+ cluster.deleteAllCollections();+ }++ @Test+ public void testGetTags() {++ CloudLegacySolrClient solrClient = (CloudLegacySolrClient) cluster.getSolrClient();+ int totalCores = 0;++ // Sum all the cores of the collection by fetching tags of all nodes.+ // We should get same number than when we created the collection+ for (JettySolrRunner runner : cluster.getJettySolrRunners()) {+ String node = runner.getNodeName();+ RemoteCallCtx ctx = new RemoteCallCtx(node, solrClient);+ NodeValueFetcher fetcher = new NodeValueFetcher();++ Set<String> requestedTags = Set.of("cores");+ fetcher.getTags(requestedTags, ctx);++ // make sure we only get the tag we asked+ assertEquals(1, ctx.tags.size());++ int coresOnNode = (Integer) ctx.tags.get("cores");+ totalCores += coresOnNode;+ }++ assertEquals(numShards * numReplicas, totalCores);+ }+}