Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

Add logs and comments to split workflow (#1027)

Nazerke Seidan0b1f12c6d973
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
index 7f78286cdab..bdd2946da40 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
@@ -80,6 +80,7 @@ import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/** SolrCloud logic for splitting a shard. It's complicated! See {@code split()} below. */
public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final int MIN_NUM_SUB_SHARDS = 2;
@@ -102,6 +103,31 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
split(state, message, results);
}
+ /**
+ * Shard splits start here and make additional requests to the host of the parent shard. The
+ * sequence of requests is as follows:
+ *
+ * <ul>
+ * <li>1. Verify that there is enough disk space to create sub-shards.
+ * <li>2. If splitByPrefix is true, make request to get prefix ranges.
+ * <li>3. If this split was attempted previously and there are lingering sub-shards, delete
+ * them.
+ * <li>4. Create sub-shards in CONSTRUCTION state.
+ * <li>5. Add an initial replica to each sub-shard.
+ * <li>6. Request that parent shard wait for children to become ACTIVE.
+ * <li>7. Execute split: either LINK or REWRITE.
+ * <li>8. Apply buffered updates to the sub-shards so they are up-to-date with parent.
+ * <li>9. Determine node placement for additional replicas (but do not create yet).
+ * <li>10. If replicationFactor is more than 1, set shard state for sub-shards to RECOVERY; else
+ * mark ACTIVE.
+ * <li>11. Create additional replicas of sub-shards.
+ * </ul>
+ *
+ * <br>
+ *
+ * <p>There is a shard split doc (dev-docs/shard-split/shard-split.adoc) on how shard split works;
+ * illustrated with diagrams.
+ */
public boolean split(ClusterState clusterState, ZkNodeProps message, NamedList<Object> results)
throws Exception {
final String asyncId = message.getStr(ASYNC);
@@ -140,6 +166,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
String splitKey = message.getStr("split.key");
DocCollection collection = clusterState.getCollection(collectionName);
+ // verify that parent shard is active; if not, throw exception
Slice parentSlice = getParentSlice(clusterState, collectionName, slice, splitKey);
if (parentSlice.getState() != Slice.State.ACTIVE) {
throw new SolrException(
@@ -152,7 +179,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
+ parentSlice.getState());
}
- // find the leader for the shard
+ // find the leader for the parent shard
Replica parentShardLeader;
try {
parentShardLeader = zkStateReader.getLeaderRetry(collectionName, slice.get(), 10000);
@@ -163,10 +190,17 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
RTimerTree t;
if (ccc.getCoreContainer().getNodeConfig().getMetricsConfig().isEnabled()) {
- t = timings.sub("checkDiskSpace");
- checkDiskSpace(
- collectionName, slice.get(), parentShardLeader, splitMethod, ccc.getSolrCloudManager());
- t.stop();
+ // check disk space for shard split
+ if (Boolean.parseBoolean(System.getProperty(SHARDSPLIT_CHECKDISKSPACE_ENABLED, "true"))) {
+ // 1. verify that there is enough space on disk to create sub-shards
+ log.debug(
+ "SplitShardCmd: verify that there is enough space on disk to create sub-shards for slice: {}",
+ parentShardLeader);
+ t = timings.sub("checkDiskSpace");
+ checkDiskSpace(
+ collectionName, slice.get(), parentShardLeader, splitMethod, ccc.getSolrCloudManager());
+ t.stop();
+ }
}
// let's record the ephemeralOwner of the parent leader node
@@ -245,6 +279,8 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
ShardHandler shardHandler = ccc.newShardHandler();
+ // 2. if split request has splitByPrefix set to true, make request to SplitOp to get prefix
+ // ranges of sub-shards
if (message.getBool(CommonAdminParams.SPLIT_BY_PREFIX, false)) {
t = timings.sub("getRanges");
@@ -253,7 +289,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
params.set(CoreAdminParams.GET_RANGES, "true");
params.set(CommonAdminParams.SPLIT_METHOD, splitMethod.toLower());
params.set(CoreAdminParams.CORE, parentShardLeader.getStr("core"));
- // Only 2 is currently supported
+ // only 2 sub-shards are currently supported
// int numSubShards = message.getInt(NUM_SUB_SHARDS, DEFAULT_NUM_SUB_SHARDS);
// params.set(NUM_SUB_SHARDS, Integer.toString(numSubShards));
@@ -302,6 +338,8 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
firstNrtReplica);
t.stop();
+ // 3. if this shard has attempted a split before and failed, there will be lingering INACTIVE
+ // sub-shards. Clean these up before proceeding
boolean oldShardsDeleted = false;
for (String subSlice : subSlices) {
Slice oSlice = collection.getSlice(subSlice);
@@ -340,6 +378,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
collection = clusterState.getCollection(collectionName);
}
+ // 4. create the child sub-shards in CONSTRUCTION state
String nodeName = parentShardLeader.getNodeName();
t = timings.sub("createSubSlicesAndLeadersInState");
@@ -377,6 +416,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
CollectionHandlingUtils.waitForNewShard(
collectionName, subSlice, ccc.getZkStateReader());
+ // 5. and add the initial replica for each sub-shard
log.debug(
"Adding first replica {} as part of slice {} of collection {} on {}",
subShardName,
@@ -414,6 +454,8 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
handleFailureOnAsyncRequest(results, msgOnError);
}
t.stop();
+
+ // 6. request that parent shard wait for children to become active
t = timings.sub("waitForSubSliceLeadersAlive");
{
final ShardRequestTracker shardRequestTracker =
@@ -458,6 +500,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
parentShardLeader);
}
+ // 7. execute actual split
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.SPLIT.toString());
params.set(CommonAdminParams.SPLIT_METHOD, splitMethod.toLower());
@@ -484,8 +527,8 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
log.debug("Index on shard: {} split into {} successfully", nodeName, subShardNames.size());
}
+ // 8. apply buffered updates on sub-shards
t = timings.sub("applyBufferedUpdates");
- // apply buffered updates on sub-shards
{
final ShardRequestTracker shardRequestTracker =
CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc);
@@ -513,8 +556,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
log.debug("Successfully applied buffered updates on : {}", subShardNames);
- // Replica creation for the new Slices
-
+ // 9. determine node placement for additional replicas
Set<String> nodes = clusterState.getLiveNodes();
List<String> nodeList = new ArrayList<>(nodes.size());
nodeList.addAll(nodes);
@@ -694,6 +736,8 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
// this ensures that the logic inside ReplicaMutator to update sub-shard state to 'active'
// always gets a chance to execute. See SOLR-7673
+ // 10. if replicationFactor > 1, set shard state for sub-shards to RECOVERY; otherwise mark
+ // ACTIVE
if (repFactor == 1) {
// A commit is needed so that documents are visible when the sub-shard replicas come up
// (Note: This commit used to be after the state switch, but was brought here before the
@@ -723,7 +767,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
ccc.offerStateUpdate(Utils.toJSON(m));
}
} else {
- log.info("Requesting shard state be set to 'recovery'");
+ log.debug("Requesting shard state be set to 'recovery' for sub-shards: {}", subSlices);
Map<String, Object> propMap = new HashMap<>();
propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower());
for (String subSlice : subSlices) {
@@ -744,7 +788,7 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
}
t = timings.sub("createCoresForReplicas");
- // now actually create replica cores on sub shard nodes
+ // 11. now actually create replica cores on sub shard nodes
for (Map<String, Object> replica : replicas) {
new AddReplicaCmd(ccc).addReplica(clusterState, new ZkNodeProps(replica), results, null);
}
@@ -774,8 +818,8 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
results.add(CommonParams.TIMING, timings.asNamedList());
}
success = true;
- // don't unlock the shard yet - only do this if the final switch-over in
- // ReplicaMutator succeeds (or fails)
+ // don't unlock the shard yet - only do this if the final switch-over in ReplicaMutator
+ // succeeds (or fails)
return true;
} catch (SolrException e) {
throw e;
@@ -814,10 +858,6 @@ public class SplitShardCmd implements CollApiCmds.CollectionApiCommand {
SolrIndexSplitter.SplitMethod method,
SolrCloudManager cloudManager)
throws SolrException {
- // check that the system property is enabled. It should not be disabled by default.
- if (!Boolean.parseBoolean(System.getProperty(SHARDSPLIT_CHECKDISKSPACE_ENABLED, "true"))) {
- return;
- }
// check that enough disk space is available on the parent leader node
// otherwise the actual index splitting will always fail
NodeStateProvider nodeStateProvider = cloudManager.getNodeStateProvider();
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.java b/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.java
index 2e3481bc966..c743b27824b 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.java
@@ -35,6 +35,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.StringHelper;
import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.cloud.ZkShardTerms;
+import org.apache.solr.cloud.api.collections.SplitShardCmd;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.CompositeIdRouter;
@@ -51,11 +52,19 @@ import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.update.SolrIndexSplitter;
import org.apache.solr.update.SplitIndexCommand;
+import org.apache.solr.update.UpdateHandler;
import org.apache.solr.util.RTimer;
import org.apache.solr.util.RefCounted;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/**
+ * CoreAdminOp implementation for shard splits. This request is enqueued when {@link SplitShardCmd}
+ * is processed. This operation handles two types of requests: 1. If {@link
+ * CommonAdminParams#SPLIT_BY_PREFIX} is true, the request to calculate document ranges for the
+ * sub-shards is processed here. 2. For any split request, the actual index split is processed here.
+ * This calls into {@link UpdateHandler#split(SplitIndexCommand)} to execute split.
+ */
class SplitOp implements CoreAdminHandler.CoreAdminOp {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -63,16 +72,18 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {
@Override
public void execute(CoreAdminHandler.CallInfo it) throws Exception {
SolrParams params = it.req.getParams();
-
String splitKey = params.get("split.key");
String[] newCoreNames = params.getParams("targetCore");
String cname = params.get(CoreAdminParams.CORE, "");
+ // if split request has splitByPrefix set to true, we will first make a request to SplitOp
+ // to calculate the prefix ranges, and do the actual split in a separate request
if (params.getBool(GET_RANGES, false)) {
handleGetRanges(it, cname);
return;
}
+ // if not using splitByPrefix, determine split partitions
List<DocRouter.Range> ranges = null;
String[] pathsArr = params.getParams(PATH);
@@ -98,6 +109,7 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {
}
}
+ // if not using splitByPrefix, ensure either path or targetCore specified
if ((pathsArr == null || pathsArr.length == 0)
&& (newCoreNames == null || newCoreNames.length == 0)) {
throw new SolrException(
@@ -124,7 +136,9 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {
DocRouter router = null;
String routeFieldName = null;
+ // if in SolrCloud mode, get collection and shard names
if (it.handler.coreContainer.isZooKeeperAware()) {
+ log.trace("SplitOp: Determine which router is associated with the shard for core");
ClusterState clusterState = it.handler.coreContainer.getZkController().getClusterState();
String collectionName =
parentCore.getCoreDescriptor().getCloudDescriptor().getCollectionName();
@@ -145,6 +159,7 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {
}
if (pathsArr == null) {
+ log.trace("SplitOp: Create array of paths for sub-shards of core");
newCores = new ArrayList<>(partitions);
for (String newCoreName : newCoreNames) {
SolrCore newcore = it.handler.coreContainer.getCore(newCoreName);
@@ -189,6 +204,7 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {
parentCore.getUpdateHandler().split(cmd);
if (it.handler.coreContainer.isZooKeeperAware()) {
+ log.trace("SplitOp: Create cloud descriptors for sub-shards of core");
for (SolrCore newcore : newCores) {
// the index of the core changed from empty to have some data, its term must be not zero
CloudDescriptor cd = newcore.getCoreDescriptor().getCloudDescriptor();
@@ -204,7 +220,7 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {
// After the split has completed, someone (here?) should start the process of replaying the
// buffered updates.
} catch (Exception e) {
- log.error("ERROR executing split:", e);
+ log.error("ERROR executing split: ", e);
throw e;
} finally {
if (req != null) req.close();