Apache Solr open-source search software
SOLR-18081: ShardRequestTracker should use a unique key for requests (#4068)
Houston Putmand766e9e72303
diff --git a/changelog/unreleased/solr-18081-shard-request-tracker-replica-name.yml b/changelog/unreleased/solr-18081-shard-request-tracker-replica-name.ymlnew file mode 100644index 00000000000..b5b390ccce6--- /dev/null+++ b/changelog/unreleased/solr-18081-shard-request-tracker-replica-name.yml@@ -0,0 +1,9 @@+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc+title: ShardRequestTracker now indexes Admin API results by node and replica rather than just node. This fixes situations where multiple sub-requests are sent to a single node.+type: fixed # added, changed, fixed, deprecated, removed, dependency_update, security, other+authors:+ - name: Houston Putman+ nick: HoustonPutman+links:+ - name: SOLR-18081+ url: https://issues.apache.org/jira/browse/SOLR-18081diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.javaindex a13bce718d8..91392c11146 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java@@ -175,7 +175,8 @@ public class AddReplicaCmd implements CollApiCmds.CollectionApiCommand {ModifiableSolrParams params =getReplicaParams(message, collectionName, coll, skipCreateReplicaInClusterState, createReplica);- shardRequestTracker.sendShardRequest(createReplica.node, params, shardHandler);+ shardRequestTracker.sendShardRequest(+ createReplica.node, createReplica.coreNodeName, params, shardHandler);}Runnable runnable =diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.javaindex 60ec1cb7e6c..04b2a1417bb 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java@@ -350,7 +350,7 @@ public class BackupCmd implements CollApiCmds.CollectionApiCommand {slice.getName(), backupManager.getBackupId().getId());params.set(CoreAdminParams.SHARD_BACKUP_ID, shardBackupId.getIdAsString());- shardRequestTracker.sendShardRequest(replica.getNodeName(), params, shardHandler);+ shardRequestTracker.sendShardRequest(replica, params, shardHandler);log.debug("Sent backup request to core={} for backupName={}", coreName, backupName);}log.debug("Sent backup requests to all shard leaders for backupName={}", backupName);@@ -532,7 +532,7 @@ public class BackupCmd implements CollApiCmds.CollectionApiCommand {params.set(CoreAdminParams.COMMIT_NAME, snapshotMeta.get().getName());}- shardRequestTracker.sendShardRequest(replica.getNodeName(), params, shardHandler);+ shardRequestTracker.sendShardRequest(replica, params, shardHandler);log.debug("Sent backup request to core={} for backupName={}", coreName, backupName);}log.debug("Sent backup requests to all shard leaders for backupName={}", backupName);diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.javaindex 454b5363841..3840683d6ef 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java@@ -314,7 +314,8 @@ public class CollApiCmds {String baseUrl = ccc.getZkStateReader().getBaseUrlForNodeName(message.getStr(NODE_NAME_PROP));ShardRequest sreq = new ShardRequest();- sreq.nodeName = message.getStr(ZkStateReader.CORE_NAME_PROP);+ sreq.nodeName = message.getStr(NODE_NAME_PROP);+ sreq.coreNodeName = message.getStr(CORE_NODE_NAME_PROP);// yes, they must use same admin handler path everywhere...params.set("qt", ccc.getAdminPath());sreq.purpose = ShardRequest.PURPOSE_PRIVATE;diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.javaindex 74232b11ddd..b51562aa5ff 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java@@ -224,7 +224,6 @@ public class CollectionHandlingUtils {String slice,Replica parentShardLeader) {log.debug("Calling soft commit to make sub shard updates visible");- String coreUrl = parentShardLeader.getCoreUrl();// HttpShardHandler is hard coded to send a QueryRequest hence we go direct// and we force open a searcher so that we have documents to show upon switching statesUpdateResponse updateResponse = null;@@ -232,13 +231,27 @@ public class CollectionHandlingUtils {updateResponse =softCommit(solrClient, parentShardLeader.getBaseUrl(), parentShardLeader.getCoreName());CollectionHandlingUtils.processResponse(- results, null, coreUrl, updateResponse, slice, Collections.emptySet());+ results,+ null,+ parentShardLeader.getNodeName(),+ parentShardLeader.getName(),+ updateResponse,+ slice,+ Collections.emptySet(),+ null);} catch (Exception e) {CollectionHandlingUtils.processResponse(- results, e, coreUrl, updateResponse, slice, Collections.emptySet());+ results,+ e,+ parentShardLeader.getNodeName(),+ parentShardLeader.getCoreName(),+ updateResponse,+ slice,+ Collections.emptySet(),+ null);throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,- "Unable to call distrib softCommit on: " + coreUrl,+ "Unable to call distrib softCommit on: " + parentShardLeader.getCoreUrl(),e);}}@@ -419,22 +432,27 @@ public class CollectionHandlingUtils {}static void processResponse(- NamedList<Object> results, ShardResponse srsp, Set<String> okayExceptions) {+ NamedList<Object> results, ShardResponse srsp, Set<String> okayExceptions, String asyncId) {Throwable e = srsp.getException();String nodeName = srsp.getNodeName();+ // Use core or coreNodeName if given as a param, otherwise use nodeName+ String coreNodeName = srsp.getShardRequest().coreNodeName;SolrResponse solrResponse = srsp.getSolrResponse();String shard = srsp.getShard();- processResponse(results, e, nodeName, solrResponse, shard, okayExceptions);+ processResponse(+ results, e, nodeName, coreNodeName, solrResponse, shard, okayExceptions, asyncId);}static void processResponse(NamedList<Object> results,Throwable e,String nodeName,+ String coreNodeName,SolrResponse solrResponse,String shard,- Set<String> okayExceptions) {+ Set<String> okayExceptions,+ String asyncId) {String rootThrowable = null;if (e instanceof RemoteSolrException remoteSolrException) {rootThrowable = remoteSolrException.getRootThrowable();@@ -442,9 +460,10 @@ public class CollectionHandlingUtils {if (e != null && (rootThrowable == null || !okayExceptions.contains(rootThrowable))) {log.error("Error from shard: {}", shard, e);- addFailure(results, nodeName, e.getClass().getName() + ":" + e.getMessage());- } else {- addSuccess(results, nodeName, solrResponse.getResponse());+ addFailure(results, nodeName, coreNodeName, e);+ } else if (asyncId == null) {+ // Do not add a success for async requests, that will be done when the async result is found+ addSuccess(results, nodeName, coreNodeName, solrResponse.getResponse());}}@@ -468,24 +487,38 @@ public class CollectionHandlingUtils {results.add("exception", nl);}- private static void addFailure(NamedList<Object> results, String key, Object value) {+ public static String requestKey(Replica replica) {+ return requestKey(replica.getNodeName(), replica.getName());+ }++ public static String requestKey(String nodeName, String coreNodeName) {+ if (coreNodeName == null) {+ return nodeName;+ } else {+ return nodeName + "/" + coreNodeName;+ }+ }++ private static void addFailure(+ NamedList<Object> results, String nodeName, String coreNodeName, Object value) {@SuppressWarnings("unchecked")SimpleOrderedMap<Object> failure = (SimpleOrderedMap<Object>) results.get("failure");if (failure == null) {failure = new SimpleOrderedMap<>();results.add("failure", failure);}- failure.add(key, value);+ failure.add(requestKey(nodeName, coreNodeName), value);}- private static void addSuccess(NamedList<Object> results, String key, Object value) {+ private static void addSuccess(+ NamedList<Object> results, String nodeName, String coreNodeName, Object value) {@SuppressWarnings("unchecked")SimpleOrderedMap<Object> success = (SimpleOrderedMap<Object>) results.get("success");if (success == null) {success = new SimpleOrderedMap<>();results.add("success", success);}- success.add(key, value);+ success.add(requestKey(nodeName, coreNodeName), value);}private static NamedList<Object> waitForCoreAdminAsyncCallToComplete(@@ -493,6 +526,7 @@ public class CollectionHandlingUtils {String adminPath,ZkStateReader zkStateReader,String nodeName,+ String coreNodeName,String requestId) {ShardHandler shardHandler = shardHandlerFactory.getShardHandler();ModifiableSolrParams params = new ModifiableSolrParams();@@ -508,6 +542,8 @@ public class CollectionHandlingUtils {sreq.shards = new String[] {replica};sreq.actualShards = sreq.shards;sreq.params = params;+ sreq.nodeName = nodeName;+ sreq.coreNodeName = coreNodeName;shardHandler.submit(sreq, replica, sreq.params);@@ -515,8 +551,6 @@ public class CollectionHandlingUtils {do {srsp = shardHandler.takeCompletedOrError();if (srsp != null) {- NamedList<Object> results = new NamedList<>();- processResponse(results, srsp, Collections.emptySet());if (srsp.getSolrResponse().getResponse() == null) {NamedList<Object> response = new NamedList<>();response.add("STATUS", "failed");@@ -524,6 +558,16 @@ public class CollectionHandlingUtils {}String r = (String) srsp.getSolrResponse().getResponse().get("STATUS");+ if (r == null) {+ // For Collections API Calls+ r = (String) srsp.getSolrResponse().getResponse()._get("status/state");+ }+ if (r == null) {+ throw new SolrException(+ SolrException.ErrorCode.SERVER_ERROR,+ "Could not find status of async command in response: "+ + srsp.getSolrResponse().getResponse().toString());+ }if (r.equals("running")) {log.debug("The task is still RUNNING, continuing to wait.");try {@@ -533,6 +577,15 @@ public class CollectionHandlingUtils {}continue;+ } else if (r.equals("submitted")) {+ log.debug("The task is still SUBMITTED, continuing to wait.");+ try {+ Thread.sleep(1000);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ }+ continue;+} else if (r.equals("completed")) {log.debug("The task is COMPLETED, returning");return srsp.getSolrResponse().getResponse();@@ -571,21 +624,28 @@ public class CollectionHandlingUtils {public static ShardRequestTracker syncRequestTracker(AdminCmdContext adminCmdContext, CollectionCommandContext ccc) {- return requestTracker(null, ccc);+ return syncRequestTracker(adminCmdContext, ccc.getAdminPath(), ccc);+ }++ public static ShardRequestTracker syncRequestTracker(+ AdminCmdContext adminCmdContext, String adminPath, CollectionCommandContext ccc) {+ return requestTracker(null, adminPath, ccc);}public static ShardRequestTracker asyncRequestTracker(AdminCmdContext adminCmdContext, CollectionCommandContext ccc) {- return requestTracker(adminCmdContext.getAsyncId(), ccc);+ return asyncRequestTracker(adminCmdContext, ccc.getAdminPath(), ccc);+ }++ public static ShardRequestTracker asyncRequestTracker(+ AdminCmdContext adminCmdContext, String adminPath, CollectionCommandContext ccc) {+ return requestTracker(adminCmdContext.getAsyncId(), adminPath, ccc);}protected static ShardRequestTracker requestTracker(- String asyncId, CollectionCommandContext ccc) {+ String asyncId, String adminPath, CollectionCommandContext ccc) {return new ShardRequestTracker(- asyncId,- ccc.getAdminPath(),- ccc.getZkStateReader(),- ccc.newShardHandler().getShardHandlerFactory());+ asyncId, adminPath, ccc.getZkStateReader(), ccc.newShardHandler().getShardHandlerFactory());}public static class ShardRequestTracker {@@ -593,7 +653,7 @@ public class CollectionHandlingUtils {private final String adminPath;private final ZkStateReader zkStateReader;private final ShardHandlerFactory shardHandlerFactory;- private final NamedList<String> shardAsyncIdByNode = new NamedList<String>();+ private final List<AsyncCmdInfo> shardAsyncCmds = new ArrayList<>();public ShardRequestTracker(String asyncId,@@ -621,14 +681,13 @@ public class CollectionHandlingUtils {for (Replica replica : slice.getReplicas()) {if ((stateMatcher == null|| Replica.State.getState(replica.getStr(ZkStateReader.STATE_PROP)) == stateMatcher)) {- if (clusterState.liveNodesContain(replica.getStr(ZkStateReader.NODE_NAME_PROP))) {+ if (clusterState.liveNodesContain(replica.getNodeName())) {// For thread safety, only simple clone the ModifiableSolrParamsModifiableSolrParams cloneParams = new ModifiableSolrParams();cloneParams.add(params);- cloneParams.set(CoreAdminParams.CORE, replica.getStr(ZkStateReader.CORE_NAME_PROP));+ cloneParams.set(CoreAdminParams.CORE, replica.getCoreName());- sendShardRequest(- replica.getStr(ZkStateReader.NODE_NAME_PROP), cloneParams, shardHandler);+ sendShardRequest(replica.getNodeName(), replica.getName(), cloneParams, shardHandler);} else {notLiveReplicas.add(replica);}@@ -638,12 +697,22 @@ public class CollectionHandlingUtils {}public void sendShardRequest(- String nodeName, ModifiableSolrParams params, ShardHandler shardHandler) {- sendShardRequest(nodeName, params, shardHandler, adminPath, zkStateReader);+ Replica replica, ModifiableSolrParams params, ShardHandler shardHandler) {+ sendShardRequest(+ replica.getNodeName(), replica.getName(), params, shardHandler, adminPath, zkStateReader);+ }++ public void sendShardRequest(+ String nodeName,+ String coreNodeName,+ ModifiableSolrParams params,+ ShardHandler shardHandler) {+ sendShardRequest(nodeName, coreNodeName, params, shardHandler, adminPath, zkStateReader);}public void sendShardRequest(String nodeName,+ String coreNodeName,ModifiableSolrParams params,ShardHandler shardHandler,String adminPath,@@ -652,7 +721,7 @@ public class CollectionHandlingUtils {String coreAdminAsyncId = asyncId + Math.abs(System.nanoTime());params.set(ASYNC, coreAdminAsyncId);// Track async requests- shardAsyncIdByNode.add(nodeName, coreAdminAsyncId);+ shardAsyncCmds.add(AsyncCmdInfo.from(nodeName, coreNodeName, coreAdminAsyncId));}ShardRequest sreq = new ShardRequest();@@ -662,6 +731,7 @@ public class CollectionHandlingUtils {sreq.shards = new String[] {replica};sreq.actualShards = sreq.shards;sreq.nodeName = nodeName;+ sreq.coreNodeName = coreNodeName;sreq.params = params;shardHandler.submit(sreq, replica, sreq.params);@@ -684,9 +754,12 @@ public class CollectionHandlingUtils {// Processes all shard responsesShardResponse srsp;do {- srsp = shardHandler.takeCompletedOrError();+ srsp =+ abortOnError+ ? shardHandler.takeCompletedOrError()+ : shardHandler.takeCompletedIncludingErrors();if (srsp != null) {- processResponse(results, srsp, okayExceptions);+ processResponse(results, srsp, okayExceptions, asyncId);Throwable exception = srsp.getException();if (abortOnError && exception != null) {// drain pending requests@@ -702,25 +775,53 @@ public class CollectionHandlingUtils {if (asyncId != null) {// TODO: Shouldn't we abort with msgOnError exception when failure?waitForAsyncCallsToComplete(results);- shardAsyncIdByNode.clear();+ shardAsyncCmds.clear();}}private void waitForAsyncCallsToComplete(NamedList<Object> results) {- for (Map.Entry<String, String> nodeToAsync : shardAsyncIdByNode) {- final String node = nodeToAsync.getKey();- final String shardAsyncId = nodeToAsync.getValue();- log.debug("I am Waiting for :{}/{}", node, shardAsyncId);+ for (AsyncCmdInfo asyncCmdInfo : shardAsyncCmds) {+ Object failure =+ results._get("failure/" + requestKey(asyncCmdInfo.nodeName, asyncCmdInfo.coreNodeName));+ // Do not wait for Async calls that have already failed+ if (failure != null) {+ return;+ }+ final String node = asyncCmdInfo.nodeName;+ final String coreNodeName = asyncCmdInfo.coreNodeName;+ final String shardAsyncId = asyncCmdInfo.asyncId;+ log.info("I am Waiting for: {}/{}/{}", node, coreNodeName, shardAsyncId);NamedList<Object> reqResult =waitForCoreAdminAsyncCallToComplete(- shardHandlerFactory, adminPath, zkStateReader, node, shardAsyncId);- if ("failed".equalsIgnoreCase(((String) reqResult.get("STATUS")))) {- log.error("Error from shard {}: {}", node, reqResult);- addFailure(results, node, reqResult);+ shardHandlerFactory, adminPath, zkStateReader, node, coreNodeName, shardAsyncId);+ String status = (String) reqResult.get("STATUS");+ if (status == null) {+ // For Collections API Calls+ status = (String) reqResult._get("status/state");+ }+ if ("failed".equalsIgnoreCase(status)) {+ log.error("Error from shard {}/{}: {}", node, coreNodeName, reqResult);+ addFailure(results, node, coreNodeName, reqResult);} else {- addSuccess(results, node, reqResult);+ addSuccess(results, node, coreNodeName, reqResult);}}}}++ private static class AsyncCmdInfo {+ protected final String nodeName;+ protected final String coreNodeName;+ protected final String asyncId;++ public AsyncCmdInfo(String nodeName, String coreNodeName, String asyncId) {+ this.nodeName = nodeName;+ this.coreNodeName = coreNodeName;+ this.asyncId = asyncId;+ }++ public static AsyncCmdInfo from(String nodeName, String coreNodeName, String asyncId) {+ return new AsyncCmdInfo(nodeName, coreNodeName, asyncId);+ }+ }}diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.javaindex f9aab00769d..ff2d92378f3 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java@@ -411,7 +411,8 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {ModifiableSolrParams params = e.getValue();String nodeName = nodeNames.get(e.getKey());params.set(CoreAdminParams.CORE_NODE_NAME, replicas.get(e.getKey()).getName());- shardRequestTracker.sendShardRequest(nodeName, params, shardHandler);+ shardRequestTracker.sendShardRequest(+ nodeName, replicas.get(e.getKey()).getCoreName(), params, shardHandler);}shardRequestTracker.processResponses(diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateSnapshotCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateSnapshotCmd.javaindex 97e24fc8279..db4918f7759 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateSnapshotCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateSnapshotCmd.java@@ -119,7 +119,7 @@ public class CreateSnapshotCmd implements CollApiCmds.CollectionApiCommand {params.set(CORE_NAME_PROP, coreName);params.set(CoreAdminParams.COMMIT_NAME, commitName);- shardRequestTracker.sendShardRequest(replica.getNodeName(), params, shardHandler);+ shardRequestTracker.sendShardRequest(replica, params, shardHandler);log.debug("Sent createsnapshot request to core={} with commitName={}", coreName, commitName);diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.javaindex 5246401db6e..a2ad2cdd5b5 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteReplicaCmd.java@@ -313,7 +313,7 @@ public class DeleteReplicaCmd implements CollectionApiCommand {final ShardRequestTracker shardRequestTracker =CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc);if (isLive) {- shardRequestTracker.sendShardRequest(replica.getNodeName(), params, shardHandler);+ shardRequestTracker.sendShardRequest(replica, params, shardHandler);}Callable<Boolean> callable =diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.javaindex ffb93c48fc2..af321c7b670 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteSnapshotCmd.java@@ -123,7 +123,7 @@ public class DeleteSnapshotCmd implements CollApiCmds.CollectionApiCommand {log.info("Sending deletesnapshot request to core={} with commitName={}", coreName, commitName);- shardRequestTracker.sendShardRequest(replica.getNodeName(), params, shardHandler);+ shardRequestTracker.sendShardRequest(replica, params, shardHandler);}}}diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.javaindex ba56c8ab9dd..2869f9866b6 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java@@ -232,7 +232,7 @@ public class MigrateCmd implements CollApiCmds.CollectionApiCommand {{final ShardRequestTracker shardRequestTracker =CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc);- shardRequestTracker.sendShardRequest(targetLeader.getNodeName(), params, shardHandler);+ shardRequestTracker.sendShardRequest(targetLeader, params, shardHandler);shardRequestTracker.processResponses(results, shardHandler, true, "MIGRATE failed to request node to buffer updates");@@ -358,7 +358,7 @@ public class MigrateCmd implements CollApiCmds.CollectionApiCommand {CollectionHandlingUtils.syncRequestTracker(adminCmdContext, ccc);// we don't want this to happen asynchronouslysyncRequestTracker.sendShardRequest(- tempSourceLeader.getNodeName(), new ModifiableSolrParams(cmd.getParams()), shardHandler);+ tempSourceLeader, new ModifiableSolrParams(cmd.getParams()), shardHandler);syncRequestTracker.processResponses(results,@@ -376,12 +376,10 @@ public class MigrateCmd implements CollApiCmds.CollectionApiCommand {params.set(CoreAdminParams.RANGES, splitRange.toString());params.set("split.key", splitKey);- String tempNodeName = sourceLeader.getNodeName();-{final ShardRequestTracker shardRequestTracker =CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc);- shardRequestTracker.sendShardRequest(tempNodeName, params, shardHandler);+ shardRequestTracker.sendShardRequest(sourceLeader, params, shardHandler);shardRequestTracker.processResponses(results, shardHandler, true, "MIGRATE failed to invoke SPLIT core admin command");}@@ -447,7 +445,7 @@ public class MigrateCmd implements CollApiCmds.CollectionApiCommand {{final ShardRequestTracker shardRequestTracker =CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc);- shardRequestTracker.sendShardRequest(tempSourceLeader.getNodeName(), params, shardHandler);+ shardRequestTracker.sendShardRequest(tempSourceLeader, params, shardHandler);shardRequestTracker.processResponses(results,@@ -468,7 +466,7 @@ public class MigrateCmd implements CollApiCmds.CollectionApiCommand {final ShardRequestTracker shardRequestTracker =CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc);- shardRequestTracker.sendShardRequest(targetLeader.getNodeName(), params, shardHandler);+ shardRequestTracker.sendShardRequest(targetLeader, params, shardHandler);String msg ="MIGRATE failed to merge "+ tempCollectionReplica2@@ -487,7 +485,7 @@ public class MigrateCmd implements CollApiCmds.CollectionApiCommand {{final ShardRequestTracker shardRequestTracker =CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc);- shardRequestTracker.sendShardRequest(targetLeader.getNodeName(), params, shardHandler);+ shardRequestTracker.sendShardRequest(targetLeader, params, shardHandler);shardRequestTracker.processResponses(results, shardHandler, true, "MIGRATE failed to request node to apply buffered updates");}diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.javaindex 3298ecba2bc..200c9a6607d 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java@@ -279,8 +279,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {{final ShardRequestTracker shardRequestTracker =CollectionHandlingUtils.syncRequestTracker(adminCmdContext, ccc);- shardRequestTracker.sendShardRequest(- parentShardLeader.getNodeName(), params, shardHandler);+ shardRequestTracker.sendShardRequest(parentShardLeader, params, shardHandler);SimpleOrderedMap<Object> getRangesResults = new SimpleOrderedMap<>();String msgOnError = "SPLITSHARD failed to invoke SPLIT.getRanges core admin command";shardRequestTracker.processResponses(getRangesResults, shardHandler, true, msgOnError);@@ -463,7 +462,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {cmd.setOnlyIfLeader(true);ModifiableSolrParams p = new ModifiableSolrParams(cmd.getParams());- shardRequestTracker.sendShardRequest(nodeName, p, shardHandler);+ shardRequestTracker.sendShardRequest(nodeName, subShardName, p, shardHandler);}String msgOnError = "SPLITSHARD timed out waiting for subshard leaders to come up";@@ -502,7 +501,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {{final ShardRequestTracker shardRequestTracker =CollectionHandlingUtils.asyncRequestTracker(adminCmdContext, ccc);- shardRequestTracker.sendShardRequest(parentShardLeader.getNodeName(), params, shardHandler);+ shardRequestTracker.sendShardRequest(parentShardLeader, params, shardHandler);String msgOnError = "SPLITSHARD failed to invoke SPLIT core admin command";shardRequestTracker.processResponses(results, shardHandler, true, msgOnError);@@ -531,7 +530,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {CoreAdminParams.CoreAdminAction.REQUESTAPPLYUPDATES.toString());params.set(CoreAdminParams.NAME, subShardName);- shardRequestTracker.sendShardRequest(nodeName, params, shardHandler);+ shardRequestTracker.sendShardRequest(nodeName, subShardName, params, shardHandler);}String msgOnError =diff --git a/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java b/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.javaindex d7d7da04f22..5222b38abee 100644--- a/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java+++ b/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java@@ -18,6 +18,7 @@ package org.apache.solr.handler.component;import java.util.ArrayList;import java.util.List;+import java.util.Map;import org.apache.solr.common.params.ModifiableSolrParams;// todo... when finalized make accessors@@ -56,6 +57,12 @@ public class ShardRequest {/** may be null */public String nodeName;+ /** may be null */+ public String coreNodeName;++ /** may be null */+ public Map<String, String> headers;+// TODO: one could store a list of numbers to correlate where returned docs// go in the top-level response rather than looking up by id...// this would work well if we ever transitioned to using internal ids anddiff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/AsyncCallRequestStatusResponseTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/AsyncCallRequestStatusResponseTest.javaindex 4c4f476e48e..ca683c4d764 100644--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/AsyncCallRequestStatusResponseTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/AsyncCallRequestStatusResponseTest.java@@ -68,8 +68,9 @@ public class AsyncCallRequestStatusResponseTest extends SolrCloudTestCase {final NamedList<?> success = (NamedList<?>) r.get("success");assertNotNull("Expected 'success' response" + r, success);- final int actualSuccessElems = 2 * (numShards * numReplicas);- // every replica responds once on submit and once on complete+ final int actualSuccessElems = numShards * numReplicas;+ // every replica responds either once on submit (failure) or once on complete (if submit+ // succeeds)assertEquals("Expected " + actualSuccessElems + " elements in the success element" + success.jsonStr(),actualSuccessElems,