Apache Solr open-source search software
Add logs and comments to split workflow (#1027)
Nazerke Seidan0b1f12c6d973
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.javaindex 7f78286cdab..bdd2946da40 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java@@ -80,6 +80,7 @@ import org.apache.zookeeper.data.Stat;import org.slf4j.Logger;import org.slf4j.LoggerFactory;+/** SolrCloud logic for splitting a shard. It's complicated! See {@code split()} below. */public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());private static final int MIN_NUM_SUB_SHARDS = 2;@@ -102,6 +103,31 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {split(state, message, results);}+ /**+ * Shard splits start here and make additional requests to the host of the parent shard. The+ * sequence of requests is as follows:+ *+ * <ul>+ * <li>1. Verify that there is enough disk space to create sub-shards.+ * <li>2. If splitByPrefix is true, make request to get prefix ranges.+ * <li>3. If this split was attempted previously and there are lingering sub-shards, delete+ * them.+ * <li>4. Create sub-shards in CONSTRUCTION state.+ * <li>5. Add an initial replica to each sub-shard.+ * <li>6. Request that parent shard wait for children to become ACTIVE.+ * <li>7. Execute split: either LINK or REWRITE.+ * <li>8. Apply buffered updates to the sub-shards so they are up-to-date with parent.+ * <li>9. Determine node placement for additional replicas (but do not create yet).+ * <li>10. If replicationFactor is more than 1, set shard state for sub-shards to RECOVERY; else+ * mark ACTIVE.+ * <li>11. Create additional replicas of sub-shards.+ * </ul>+ *+ * <br>+ *+ * <p>There is a shard split doc (dev-docs/shard-split/shard-split.adoc) on how shard split works;+ * illustrated with diagrams.+ */public boolean split(ClusterState clusterState, ZkNodeProps message, NamedList<Object> results)throws Exception {final String asyncId = message.getStr(ASYNC);@@ -140,6 +166,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {String splitKey = message.getStr("split.key");DocCollection collection = clusterState.getCollection(collectionName);+ // verify that parent shard is active; if not, throw exceptionSlice parentSlice = getParentSlice(clusterState, collectionName, slice, splitKey);if (parentSlice.getState() != Slice.State.ACTIVE) {throw new SolrException(@@ -152,7 +179,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {+ parentSlice.getState());}- // find the leader for the shard+ // find the leader for the parent shardReplica parentShardLeader;try {parentShardLeader = zkStateReader.getLeaderRetry(collectionName, slice.get(), 10000);@@ -163,10 +190,17 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {RTimerTree t;if (ccc.getCoreContainer().getNodeConfig().getMetricsConfig().isEnabled()) {- t = timings.sub("checkDiskSpace");- checkDiskSpace(- collectionName, slice.get(), parentShardLeader, splitMethod, ccc.getSolrCloudManager());- t.stop();+ // check disk space for shard split+ if (Boolean.parseBoolean(System.getProperty(SHARDSPLIT_CHECKDISKSPACE_ENABLED, "true"))) {+ // 1. verify that there is enough space on disk to create sub-shards+ log.debug(+ "SplitShardCmd: verify that there is enough space on disk to create sub-shards for slice: {}",+ parentShardLeader);+ t = timings.sub("checkDiskSpace");+ checkDiskSpace(+ collectionName, slice.get(), parentShardLeader, splitMethod, ccc.getSolrCloudManager());+ t.stop();+ }}// let's record the ephemeralOwner of the parent leader node@@ -245,6 +279,8 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {ShardHandler shardHandler = ccc.newShardHandler();+ // 2. if split request has splitByPrefix set to true, make request to SplitOp to get prefix+ // ranges of sub-shardsif (message.getBool(CommonAdminParams.SPLIT_BY_PREFIX, false)) {t = timings.sub("getRanges");@@ -253,7 +289,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {params.set(CoreAdminParams.GET_RANGES, "true");params.set(CommonAdminParams.SPLIT_METHOD, splitMethod.toLower());params.set(CoreAdminParams.CORE, parentShardLeader.getStr("core"));- // Only 2 is currently supported+ // only 2 sub-shards are currently supported// int numSubShards = message.getInt(NUM_SUB_SHARDS, DEFAULT_NUM_SUB_SHARDS);// params.set(NUM_SUB_SHARDS, Integer.toString(numSubShards));@@ -302,6 +338,8 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {firstNrtReplica);t.stop();+ // 3. if this shard has attempted a split before and failed, there will be lingering INACTIVE+ // sub-shards. Clean these up before proceedingboolean oldShardsDeleted = false;for (String subSlice : subSlices) {Slice oSlice = collection.getSlice(subSlice);@@ -340,6 +378,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {collection = clusterState.getCollection(collectionName);}+ // 4. create the child sub-shards in CONSTRUCTION stateString nodeName = parentShardLeader.getNodeName();t = timings.sub("createSubSlicesAndLeadersInState");@@ -377,6 +416,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {CollectionHandlingUtils.waitForNewShard(collectionName, subSlice, ccc.getZkStateReader());+ // 5. and add the initial replica for each sub-shardlog.debug("Adding first replica {} as part of slice {} of collection {} on {}",subShardName,@@ -414,6 +454,8 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {handleFailureOnAsyncRequest(results, msgOnError);}t.stop();++ // 6. request that parent shard wait for children to become activet = timings.sub("waitForSubSliceLeadersAlive");{final ShardRequestTracker shardRequestTracker =@@ -458,6 +500,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {parentShardLeader);}+ // 7. execute actual splitModifiableSolrParams params = new ModifiableSolrParams();params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.SPLIT.toString());params.set(CommonAdminParams.SPLIT_METHOD, splitMethod.toLower());@@ -484,8 +527,8 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {log.debug("Index on shard: {} split into {} successfully", nodeName, subShardNames.size());}+ // 8. apply buffered updates on sub-shardst = timings.sub("applyBufferedUpdates");- // apply buffered updates on sub-shards{final ShardRequestTracker shardRequestTracker =CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc);@@ -513,8 +556,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {log.debug("Successfully applied buffered updates on : {}", subShardNames);- // Replica creation for the new Slices-+ // 9. determine node placement for additional replicasSet<String> nodes = clusterState.getLiveNodes();List<String> nodeList = new ArrayList<>(nodes.size());nodeList.addAll(nodes);@@ -694,6 +736,8 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {// this ensures that the logic inside ReplicaMutator to update sub-shard state to 'active'// always gets a chance to execute. See SOLR-7673+ // 10. if replicationFactor > 1, set shard state for sub-shards to RECOVERY; otherwise mark+ // ACTIVEif (repFactor == 1) {// A commit is needed so that documents are visible when the sub-shard replicas come up// (Note: This commit used to be after the state switch, but was brought here before the@@ -723,7 +767,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {ccc.offerStateUpdate(Utils.toJSON(m));}} else {- log.info("Requesting shard state be set to 'recovery'");+ log.debug("Requesting shard state be set to 'recovery' for sub-shards: {}", subSlices);Map<String, Object> propMap = new HashMap<>();propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower());for (String subSlice : subSlices) {@@ -744,7 +788,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {}t = timings.sub("createCoresForReplicas");- // now actually create replica cores on sub shard nodes+ // 11. now actually create replica cores on sub shard nodesfor (Map<String, Object> replica : replicas) {new AddReplicaCmd(ccc).addReplica(clusterState, new ZkNodeProps(replica), results, null);}@@ -774,8 +818,8 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {results.add(CommonParams.TIMING, timings.asNamedList());}success = true;- // don't unlock the shard yet - only do this if the final switch-over in- // ReplicaMutator succeeds (or fails)+ // don't unlock the shard yet - only do this if the final switch-over in ReplicaMutator+ // succeeds (or fails)return true;} catch (SolrException e) {throw e;@@ -814,10 +858,6 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {SolrIndexSplitter.SplitMethod method,SolrCloudManager cloudManager)throws SolrException {- // check that the system property is enabled. It should not be disabled by default.- if (!Boolean.parseBoolean(System.getProperty(SHARDSPLIT_CHECKDISKSPACE_ENABLED, "true"))) {- return;- }// check that enough disk space is available on the parent leader node// otherwise the actual index splitting will always failNodeStateProvider nodeStateProvider = cloudManager.getNodeStateProvider();diff --git a/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.java b/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.javaindex 2e3481bc966..c743b27824b 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.java@@ -35,6 +35,7 @@ import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.StringHelper;import org.apache.solr.cloud.CloudDescriptor;import org.apache.solr.cloud.ZkShardTerms;+import org.apache.solr.cloud.api.collections.SplitShardCmd;import org.apache.solr.common.SolrException;import org.apache.solr.common.cloud.ClusterState;import org.apache.solr.common.cloud.CompositeIdRouter;@@ -51,11 +52,19 @@ import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.search.SolrIndexSearcher;import org.apache.solr.update.SolrIndexSplitter;import org.apache.solr.update.SplitIndexCommand;+import org.apache.solr.update.UpdateHandler;import org.apache.solr.util.RTimer;import org.apache.solr.util.RefCounted;import org.slf4j.Logger;import org.slf4j.LoggerFactory;+/**+ * CoreAdminOp implementation for shard splits. This request is enqueued when {@link SplitShardCmd}+ * is processed. This operation handles two types of requests: 1. If {@link+ * CommonAdminParams#SPLIT_BY_PREFIX} is true, the request to calculate document ranges for the+ * sub-shards is processed here. 2. For any split request, the actual index split is processed here.+ * This calls into {@link UpdateHandler#split(SplitIndexCommand)} to execute split.+ */class SplitOp implements CoreAdminHandler.CoreAdminOp {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());@@ -63,16 +72,18 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {@Overridepublic void execute(CoreAdminHandler.CallInfo it) throws Exception {SolrParams params = it.req.getParams();-String splitKey = params.get("split.key");String[] newCoreNames = params.getParams("targetCore");String cname = params.get(CoreAdminParams.CORE, "");+ // if split request has splitByPrefix set to true, we will first make a request to SplitOp+ // to calculate the prefix ranges, and do the actual split in a separate requestif (params.getBool(GET_RANGES, false)) {handleGetRanges(it, cname);return;}+ // if not using splitByPrefix, determine split partitionsList<DocRouter.Range> ranges = null;String[] pathsArr = params.getParams(PATH);@@ -98,6 +109,7 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {}}+ // if not using splitByPrefix, ensure either path or targetCore specifiedif ((pathsArr == null || pathsArr.length == 0)&& (newCoreNames == null || newCoreNames.length == 0)) {throw new SolrException(@@ -124,7 +136,9 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {DocRouter router = null;String routeFieldName = null;+ // if in SolrCloud mode, get collection and shard namesif (it.handler.coreContainer.isZooKeeperAware()) {+ log.trace("SplitOp: Determine which router is associated with the shard for core");ClusterState clusterState = it.handler.coreContainer.getZkController().getClusterState();String collectionName =parentCore.getCoreDescriptor().getCloudDescriptor().getCollectionName();@@ -145,6 +159,7 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {}if (pathsArr == null) {+ log.trace("SplitOp: Create array of paths for sub-shards of core");newCores = new ArrayList<>(partitions);for (String newCoreName : newCoreNames) {SolrCore newcore = it.handler.coreContainer.getCore(newCoreName);@@ -189,6 +204,7 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {parentCore.getUpdateHandler().split(cmd);if (it.handler.coreContainer.isZooKeeperAware()) {+ log.trace("SplitOp: Create cloud descriptors for sub-shards of core");for (SolrCore newcore : newCores) {// the index of the core changed from empty to have some data, its term must be not zeroCloudDescriptor cd = newcore.getCoreDescriptor().getCloudDescriptor();@@ -204,7 +220,7 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {// After the split has completed, someone (here?) should start the process of replaying the// buffered updates.} catch (Exception e) {- log.error("ERROR executing split:", e);+ log.error("ERROR executing split: ", e);throw e;} finally {if (req != null) req.close();