Apache Solr open-source search software
SOLR-17955 & SOLR-17437: OTEL metrics: SplitShardCmd.checkDiskSpace needs conversion (#3859)
Matthew Biscocho70d8caaa7498
diff --git a/changelog/unreleased/SOLR-17437-shard-split-disk-space-check.yml b/changelog/unreleased/SOLR-17437-shard-split-disk-space-check.ymlnew file mode 100644index 00000000000..f363c311dad--- /dev/null+++ b/changelog/unreleased/SOLR-17437-shard-split-disk-space-check.yml@@ -0,0 +1,9 @@+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc+title: Fix disk space check in shard split operation+type: fixed+authors:+ - name: Matthew Biscocho+ - name: David Smiley+links:+ - name: SOLR-17437+ url: https://issues.apache.org/jira/browse/SOLR-17437diff --git a/changelog/unreleased/SOLR-17955-SplitShardCmd.checkDiskSpace-otel.yml b/changelog/unreleased/SOLR-17955-SplitShardCmd.checkDiskSpace-otel.ymlnew file mode 100644index 00000000000..cba54573654--- /dev/null+++ b/changelog/unreleased/SOLR-17955-SplitShardCmd.checkDiskSpace-otel.yml@@ -0,0 +1,9 @@+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc+title: OTEL metrics - SplitShardCmd.checkDiskSpace needs conversion+type: fixed+authors:+ - name: Matthew Biscocho+ - name: David Smiley+links:+ - name: SOLR-17955+ url: https://issues.apache.org/jira/browse/SOLR-17955\ No newline at end of filediff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.javaindex c2f0fd3d500..3298ecba2bc 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java@@ -17,6 +17,7 @@package org.apache.solr.cloud.api.collections;+import static org.apache.solr.client.solrj.response.InputStreamResponseParser.STREAM_KEY;import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE;import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;@@ -26,6 +27,7 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.CRimport static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETESHARD;import static org.apache.solr.common.params.CommonAdminParams.NUM_SUB_SHARDS;+import java.io.InputStream;import java.lang.invoke.MethodHandles;import java.util.ArrayList;import java.util.Collection;@@ -38,12 +40,14 @@ import java.util.NoSuchElementException;import java.util.Set;import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicReference;-import org.apache.solr.client.solrj.SolrResponse;import org.apache.solr.client.solrj.cloud.DistribStateManager;import org.apache.solr.client.solrj.cloud.SolrCloudManager;import org.apache.solr.client.solrj.cloud.VersionedData;+import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;+import org.apache.solr.client.solrj.impl.NodeValueFetcher;import org.apache.solr.client.solrj.request.CoreAdminRequest;import org.apache.solr.client.solrj.request.MetricsRequest;+import org.apache.solr.client.solrj.response.InputStreamResponseParser;import org.apache.solr.cloud.DistributedClusterStateUpdater;import org.apache.solr.cloud.Overseer;import org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ShardRequestTracker;@@ -858,42 +862,68 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {SolrIndexSplitter.SplitMethod method,SolrCloudManager cloudManager)throws Exception {- if (true) {- log.warn("checkDiskSpace disabled SOLR-17458 SOLR-17955");- return;- }// check that enough disk space is available on the parent leader node// otherwise the actual index splitting will always fail- String replicaName = Utils.parseMetricsReplicaName(collection, parentShardLeader.getCoreName());- String indexSizeMetricName =- "solr.core." + collection + "." + shard + "." + replicaName + ":INDEX.sizeInBytes";- String freeDiskSpaceMetricName = "solr.node:CONTAINER.fs.usableSpace";+ String indexSizeMetric = "solr_core_index_size_megabytes";+ String freeDiskSpaceMetric = "solr_disk_space_megabytes";+ String coreLabel =+ collection+ + "_"+ + shard+ + "_"+ + Utils.parseMetricsReplicaName(collection, parentShardLeader.getCoreName());ModifiableSolrParams params =- new ModifiableSolrParams()- .add("key", indexSizeMetricName)- .add("key", freeDiskSpaceMetricName);- SolrResponse rsp = new MetricsRequest(params).process(cloudManager.getSolrClient());-- Number size = (Number) rsp.getResponse()._get(List.of("metrics", indexSizeMetricName), null);- if (size == null) {- log.warn("cannot verify information for parent shard leader");- return;+ new ModifiableSolrParams().add("name", indexSizeMetric).add("name", freeDiskSpaceMetric);++ var req = new MetricsRequest(params);+ req.setResponseParser(new InputStreamResponseParser("prometheus"));++ var cloudClient = (CloudHttp2SolrClient) cloudManager.getSolrClient();+ var httpClient = cloudClient.getHttpClient();++ NamedList<Object> resp =+ httpClient.requestWithBaseUrl(parentShardLeader.getBaseUrl(), req, null);++ var indexSizeRef = new AtomicReference<Double>(-1.0);+ var freeSizeRef = new AtomicReference<Double>(-1.0);+ try (InputStream prometheusStream = (InputStream) resp.get(STREAM_KEY);+ var lines = NodeValueFetcher.Metrics.prometheusMetricStream(prometheusStream)) {++ lines+ .filter(line -> !line.isBlank() && !line.startsWith("#"))+ .forEach(+ line -> {+ if (line.contains(indexSizeMetric) && line.contains(coreLabel)) {+ indexSizeRef.set(NodeValueFetcher.Metrics.extractPrometheusValue(line));+ } else if (line.contains(freeDiskSpaceMetric) && line.contains("usable_space")) {+ freeSizeRef.set(NodeValueFetcher.Metrics.extractPrometheusValue(line));+ }+ });}- double indexSize = size.doubleValue();- Number freeSize =- (Number) rsp.getResponse()._get(List.of("metrics", freeDiskSpaceMetricName), null);- if (freeSize == null) {- log.warn("missing node disk space information for parent shard leader");- return;+ double indexSize = indexSizeRef.get();+ double freeSize = freeSizeRef.get();++ if (indexSize == -1.0) {+ throw new SolrException(+ SolrException.ErrorCode.SERVER_ERROR,+ "cannot verify index size information for parent shard leader on node "+ + parentShardLeader.getNodeName());+ }++ if (freeSize == -1.0) {+ throw new SolrException(+ SolrException.ErrorCode.SERVER_ERROR,+ "missing node disk space information for parent shard leader on node "+ + parentShardLeader.getNodeName());}// 100% more for REWRITE, 5% more for LINKdouble neededSpace =method == SolrIndexSplitter.SplitMethod.REWRITE ? 2.0 * indexSize : 1.05 * indexSize;- if (freeSize.doubleValue() < neededSpace) {+ if (freeSize < neededSpace) {throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,"not enough free disk space to perform index split on node "