Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-14928: add exponential backoff wait time when Compare And Swap fails in distributed cluster state update due to concurrent update (#2438)

Ilan Ginzburg1fff17469093
diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedClusterStateUpdater.java b/solr/core/src/java/org/apache/solr/cloud/DistributedClusterStateUpdater.java
index f57cc314afe..e48b7ced014 100644
--- a/solr/core/src/java/org/apache/solr/cloud/DistributedClusterStateUpdater.java
+++ b/solr/core/src/java/org/apache/solr/cloud/DistributedClusterStateUpdater.java
@@ -18,6 +18,7 @@
package org.apache.solr.cloud;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
+import org.apache.solr.cloud.api.collections.CollectionHandlingUtils;
import org.apache.solr.cloud.overseer.*;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.*;
@@ -446,9 +447,11 @@ public class DistributedClusterStateUpdater {
}
// We've tried to update an existing state.json and got a BadVersionException. We'll try again a few times.
// When only two threads compete, no point in waiting: if we lost this time we'll get it next time right away.
- // But if more threads compete, then waiting a bit (random delay) can improve our chances. The delay should likely
- // be proportional to the time between reading the cluster state and updating it. We can measure it in the loop above.
+ // But if more threads compete, then waiting a bit (random delay) can improve our chances. The delay should in
+ // theory grow as the number of concurrent threads attempting updates increase, but we don't know that number, so
+ // doing exponential backoff instead.
// With "per replica states" collections, concurrent attempts of even just two threads are expected to be extremely rare.
+ Thread.sleep(CollectionHandlingUtils.RANDOM.nextInt(attempt < 13 ? 1 << attempt : 1 << 13)); // max wait 2^13ms=8.192 sec
}
// We made quite a few attempts but failed repeatedly. This is pretty bad but we can't loop trying forever.
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java
index 439d7c7d690..77075664c45 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionHandlingUtils.java
@@ -111,7 +111,7 @@ public class CollectionHandlingUtils {
DocCollection.PER_REPLICA_STATE, null,
ZkStateReader.PULL_REPLICAS, "0"));
- protected static final Random RANDOM;
+ public static final Random RANDOM;
static {
// We try to make things reproducible in the context of our tests by initializing the random instance
// based on the current seed