Apache Solr open-source search software
SOLR-17102: Replaced VersionBucket array with locks on-demand (#2548)
David Smiley7ccf4d3c9af7
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 791094f6b65..cdedd6d51a3 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -115,6 +115,9 @@ Optimizationsbeginning?) in which its collection state cache was not being used, resulting in many extrarequests to Solr for cluster information. (Aparna Suresh, shalin, David Smiley)+* SOLR-17102: The VersionBucket indexing lock mechanism was replaced with something just as fast yet+ that which consumes almost no memory, saving 1MB of memory per SolrCore. (David Smiley)+Bug Fixes---------------------(No changes)diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.javaindex ed0b2a48654..86c75c7ed77 100644--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java@@ -63,6 +63,7 @@ import org.apache.lucene.index.CorruptIndexException;import org.apache.lucene.index.IndexWriter;import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.store.Directory;+import org.apache.lucene.util.BytesRef;import org.apache.solr.api.ClusterPluginsSource;import org.apache.solr.api.ContainerPluginsRegistry;import org.apache.solr.api.JerseyResource;@@ -239,7 +240,7 @@ public class CoreContainer {ExecutorUtil.newMDCAwareCachedThreadPool(new SolrNamedThreadFactory("coreContainerWorkExecutor"));- private final OrderedExecutor replayUpdatesExecutor;+ private final OrderedExecutor<BytesRef> replayUpdatesExecutor;protected volatile LogWatcher<?> logging = null;@@ -420,7 +421,7 @@ public class CoreContainer {this.containerProperties = new Properties(config.getSolrProperties());this.asyncSolrCoreLoad = asyncSolrCoreLoad;this.replayUpdatesExecutor =- new OrderedExecutor(+ new OrderedExecutor<>(cfg.getReplayUpdatesThreads(),ExecutorUtil.newMDCAwareCachedThreadPool(cfg.getReplayUpdatesThreads(), // thread count@@ -727,7 +728,7 @@ public class CoreContainer {return tracer;}- public OrderedExecutor getReplayUpdatesExecutor() {+ public OrderedExecutor<BytesRef> getReplayUpdatesExecutor() {return replayUpdatesExecutor;}diff --git a/solr/core/src/java/org/apache/solr/update/TimedVersionBucket.java b/solr/core/src/java/org/apache/solr/update/TimedVersionBucket.javadeleted file mode 100644index cdddede5a79..00000000000--- a/solr/core/src/java/org/apache/solr/update/TimedVersionBucket.java+++ /dev/null@@ -1,83 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.update;--import java.io.IOException;-import java.util.concurrent.TimeUnit;-import java.util.concurrent.locks.Condition;-import java.util.concurrent.locks.Lock;-import java.util.concurrent.locks.ReentrantLock;-import org.apache.solr.common.SolrException;--/**- * This implementation uses lock and condition and will throw exception if it can't obtain the lock- * within <code>lockTimeoutMs</code>.- *- * @lucene.internal- */-public class TimedVersionBucket extends VersionBucket {-- private final Lock lock = new ReentrantLock(true);- private final Condition condition = lock.newCondition();-- /**- * This will run the function with the lock. It will throw exception if it can't obtain the lock- * within <code>lockTimeoutMs</code>.- */- @Override- public <T, R> R runWithLock(int lockTimeoutMs, CheckedFunction<T, R> function)- throws IOException {- if (tryLock(lockTimeoutMs)) {- return function.apply();- } else {- throw new SolrException(- SolrException.ErrorCode.SERVER_ERROR,- "Unable to get version bucket lock in " + lockTimeoutMs + " ms");- }- }-- @Override- public void unlock() {- lock.unlock();- }-- @Override- public void signalAll() {- condition.signalAll();- }-- @Override- public void awaitNanos(long nanosTimeout) {- try {- if (nanosTimeout > 0) {- condition.awaitNanos(nanosTimeout);- }- } catch (InterruptedException e) {- Thread.currentThread().interrupt();- throw new RuntimeException(e);- }- }-- protected boolean tryLock(int lockTimeoutMs) {- try {- return lock.tryLock(lockTimeoutMs, TimeUnit.MILLISECONDS);- } catch (InterruptedException e) {- Thread.currentThread().interrupt();- throw new RuntimeException(e);- }- }-}diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLocks.java b/solr/core/src/java/org/apache/solr/update/UpdateLocks.javanew file mode 100644index 00000000000..5fbd27c422c--- /dev/null+++ b/solr/core/src/java/org/apache/solr/update/UpdateLocks.java@@ -0,0 +1,192 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.update;++import com.carrotsearch.hppc.IntObjectHashMap;+import java.io.IOException;+import java.util.ArrayDeque;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.locks.Condition;+import java.util.concurrent.locks.Lock;+import java.util.concurrent.locks.ReadWriteLock;+import java.util.concurrent.locks.ReentrantLock;+import java.util.concurrent.locks.ReentrantReadWriteLock;+import org.apache.lucene.util.BytesRef;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.SolrException.ErrorCode;+import org.apache.solr.util.IOFunction;++/**+ * Locks associated with updates in connection with the {@link UpdateLog}.+ *+ * @lucene.internal+ */+public class UpdateLocks {++ private final long docLockTimeoutMs;++ private final ReadWriteLock blockUpdatesLock = new ReentrantReadWriteLock(true);++ // SolrCloud's first approach was a fixed size array of locks (lock striping) using an ID's hash.+ // Sized too small, there was too much lock sharing; sized too big, there was memory waste.+ // Here we have a Map keyed by hash and a pool of locks to re-use. Synchronization is needed 2x.+ // Note: ConcurrentHashMap was also explored but HPPC came out on top, probably because+ // we can use a hashcode directly as the key, and it's GC friendly (zero-allocation).++ /** Maps a ID hashcode to a lock. Synchronize to manipulate. */+ private final IntObjectHashMap<LockAndCondition> hashToLock =+ new IntObjectHashMap<>(32) {+ @Override+ protected int hashKey(int key) {+ return key; // our keys are themselves hash-codes+ }+ };++ /** A pool of locks to avoid creating & GC'ing them too much. Must synchronize on hashToLock. */+ private final ArrayDeque<LockAndCondition> lockPool = new ArrayDeque<>(16);++ public UpdateLocks(long docLockTimeoutMs) {+ this.docLockTimeoutMs = docLockTimeoutMs;+ }++ /**+ * Acquires a lock for the given doc ID, executes the function, and releases the lock. The+ * provided {@link Condition} can be used for wait/notify control. The per-doc locking is needed+ * indirectly due to SolrCloud's internal out-of-order versioned update design.+ */+ public <R> R runWithLock(BytesRef id, IOFunction<Condition, R> function) throws IOException {+ final var startTimeNanos = System.nanoTime();++ lockForUpdate();+ try {+ // note: if we didn't need a Condition, then we could have reused+ // OrderedExecutor.SparseStripedLock over here, which is also a mechanism invented for+ // per-doc locking.++ // hashToLock isn't concurrent, but we synchronize on it briefly twice to do cheap work++ final int hash = id.hashCode();+ final LockAndCondition lock;+ // get or insert lock, increment refcount+ synchronized (hashToLock) {+ final int idx = hashToLock.indexOf(hash);+ if (hashToLock.indexExists(idx)) {+ lock = hashToLock.indexGet(idx);+ assert lock.refCount >= 1;+ lock.refCount++;+ } else {+ lock = borrowLock();+ hashToLock.indexInsert(idx, hash, lock);+ }+ }++ // try-finally ensuring we decrement the refCount+ try {+ return runWithLockInternal(id, function, lock, startTimeNanos);+ } finally {+ // decrement refcount, remove lock if unreferenced+ synchronized (hashToLock) {+ assert lock.refCount > 0; // because we incremented it+ if (--lock.refCount == 0) { // typical+ hashToLock.remove(hash);+ returnLock(lock);+ }+ }+ }++ } finally {+ unlockForUpdate();+ }+ }++ private LockAndCondition borrowLock() {+ assert Thread.holdsLock(hashToLock);+ if (lockPool.isEmpty()) {+ return new LockAndCondition();+ } else {+ return lockPool.removeLast();+ }+ }++ private void returnLock(LockAndCondition lock) {+ assert Thread.holdsLock(hashToLock);+ if (lockPool.size() < 16) {+ lockPool.add(lock);+ lock.refCount = 1; // ready for next use+ }+ }++ private <R> R runWithLockInternal(+ BytesRef id, IOFunction<Condition, R> function, LockAndCondition lock, long startTimeNanos)+ throws IOException {+ // Acquire the lock+ try {+ if (docLockTimeoutMs == 0) {+ lock.lock.lockInterruptibly();+ } else {+ long remainingNs =+ TimeUnit.MILLISECONDS.toNanos(docLockTimeoutMs) - (System.nanoTime() - startTimeNanos);+ boolean timedOut = !lock.lock.tryLock(remainingNs, TimeUnit.NANOSECONDS);+ if (timedOut) {+ throw new SolrException(+ ErrorCode.SERVER_ERROR,+ "Unable to lock doc " + id + " in " + docLockTimeoutMs + " ms");+ }+ }+ } catch (InterruptedException e) {+ // don't set interrupt status; we're ending the request+ throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to lock doc " + id, e);+ }+ // try-finally ensuring we unlock+ try {+ // We have the lock; do stuff with it+ return function.apply(lock.condition);+ } finally {+ // Release the lock+ lock.lock.unlock();+ }+ }++ private static class LockAndCondition {+ final Lock lock;+ final Condition condition;+ int refCount; // only access when synchronized on hashToLock++ LockAndCondition() {+ lock = new ReentrantLock(true); // fair+ condition = lock.newCondition();+ refCount = 1;+ }+ }++ public void lockForUpdate() {+ blockUpdatesLock.readLock().lock();+ }++ public void unlockForUpdate() {+ blockUpdatesLock.readLock().unlock();+ }++ public void blockUpdates() {+ blockUpdatesLock.writeLock().lock();+ }++ public void unblockUpdates() {+ blockUpdatesLock.writeLock().unlock();+ }+}diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.javaindex 4ebe4f311d1..9bc685f276b 100644--- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java+++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java@@ -66,6 +66,7 @@ import org.apache.solr.common.SolrInputDocument;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.CollectionUtil;+import org.apache.solr.common.util.EnvUtils;import org.apache.solr.common.util.ExecutorUtil;import org.apache.solr.common.util.IOUtils;import org.apache.solr.common.util.SolrNamedThreadFactory;@@ -223,15 +224,14 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {protected final int numDeletesByQueryToKeep = 100;protected int numRecordsToKeep;protected int maxNumLogsToKeep;- // This should only be used to initialize VersionInfo... the actual number of buckets may be- // rounded up to a power of two.- protected int numVersionBuckets;protected boolean existOldBufferLog = false;// keep track of deletes only... this is not updated on an addprotected LinkedHashMap<BytesRef, LogPtr> oldDeletes =new OldDeletesLinkedHashMap(this.numDeletesToKeep);+ private UpdateLocks updateLocks;+/** Holds the query and the version for a DeleteByQuery command */public static class DBQ {public String q; // the query string@@ -331,6 +331,10 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {return versionInfo;}+ public UpdateLocks getLocks() {+ return updateLocks;+ }+public int getNumRecordsToKeep() {return numRecordsToKeep;}@@ -339,10 +343,6 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {return maxNumLogsToKeep;}- public int getNumVersionBuckets() {- return numVersionBuckets;- }-protected static int objToInt(Object obj, int def) {if (obj != null) {return Integer.parseInt(obj.toString());@@ -381,19 +381,25 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {numRecordsToKeep = objToInt(info.initArgs.get("numRecordsToKeep"), 100);maxNumLogsToKeep = objToInt(info.initArgs.get("maxNumLogsToKeep"), 10);- numVersionBuckets = objToInt(info.initArgs.get("numVersionBuckets"), 65536);- if (numVersionBuckets <= 0)- throw new SolrException(- SolrException.ErrorCode.SERVER_ERROR,- "Number of version buckets must be greater than 0!");+ if (info.initArgs.get("numVersionBuckets") != null) {+ log.warn("numVersionBuckets is obsolete");+ }++ if (info.initArgs.get("versionBucketLockTimeoutMs") != null) {+ log.warn("versionBucketLockTimeoutMs is mostly replaced by docLockTimeoutMs");+ }+ int timeoutMs =+ objToInt(+ info.initArgs.get("docLockTimeoutMs", info.initArgs.get("versionBucketLockTimeoutMs")),+ EnvUtils.getPropertyAsLong("solr.update.docLockTimeoutMs", 0L).intValue());+ updateLocks = new UpdateLocks(timeoutMs);log.info(- "Initializing UpdateLog: dataDir={} defaultSyncLevel={} numRecordsToKeep={} maxNumLogsToKeep={} numVersionBuckets={}",+ "Initializing UpdateLog: dataDir={} defaultSyncLevel={} numRecordsToKeep={} maxNumLogsToKeep={}",dataDir,defaultSyncLevel,numRecordsToKeep,- maxNumLogsToKeep,- numVersionBuckets);+ maxNumLogsToKeep);}private final AtomicBoolean initialized = new AtomicBoolean();@@ -443,7 +449,6 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {getTlogDir(),id);}- versionInfo.reload();core.getCoreMetricManager().registerMetricProducer(SolrInfoBean.Category.TLOG.toString(), this);@@ -464,7 +469,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {usableForChildDocs = core.getLatestSchema().isUsableForChildDocs();try {- versionInfo = new VersionInfo(this, numVersionBuckets);+ versionInfo = new VersionInfo(this);} catch (SolrException e) {log.error("Unable to use updateLog: ", e);throw new SolrException(@@ -1373,7 +1378,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {ExecutorCompletionService<RecoveryInfo> cs = new ExecutorCompletionService<>(recoveryExecutor);LogReplayer replayer = new LogReplayer(recoverLogs, false);- versionInfo.blockUpdates();+ updateLocks.blockUpdates();try {state = State.REPLAYING;@@ -1383,7 +1388,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {deleteByQueries.clear();oldDeletes.clear();} finally {- versionInfo.unblockUpdates();+ updateLocks.unblockUpdates();}// At this point, we are guaranteed that any new updates coming in will see the state as@@ -1409,11 +1414,11 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {ExecutorCompletionService<RecoveryInfo> cs = new ExecutorCompletionService<>(recoveryExecutor);LogReplayer replayer = new LogReplayer(Collections.singletonList(tlog), false, true);- versionInfo.blockUpdates();+ updateLocks.blockUpdates();try {state = State.REPLAYING;} finally {- versionInfo.unblockUpdates();+ updateLocks.unblockUpdates();}return cs.submit(replayer, recoveryInfo);@@ -1427,7 +1432,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {* @param cuc any updates that have version larger than the version of cuc will be copied over*/public void copyOverBufferingUpdates(CommitUpdateCommand cuc) {- versionInfo.blockUpdates();+ updateLocks.blockUpdates();try {synchronized (this) {state = State.ACTIVE;@@ -1440,7 +1445,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {dropBufferTlog();}} finally {- versionInfo.unblockUpdates();+ updateLocks.unblockUpdates();}}@@ -1451,7 +1456,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {* @param cuc any updates that have version larger than the version of cuc will be copied over*/public void commitAndSwitchToNewTlog(CommitUpdateCommand cuc) {- versionInfo.blockUpdates();+ updateLocks.blockUpdates();try {synchronized (this) {if (tlog == null) {@@ -1465,7 +1470,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {}}} finally {- versionInfo.unblockUpdates();+ updateLocks.unblockUpdates();}}@@ -1888,7 +1893,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {// block all updates to eliminate race conditions// reading state and acting on it in the distributed update processor- versionInfo.blockUpdates();+ updateLocks.blockUpdates();try {if (state != State.ACTIVE && state != State.BUFFERING) {// we don't currently have support for handling other states@@ -1906,13 +1911,13 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {state = State.BUFFERING;} finally {- versionInfo.unblockUpdates();+ updateLocks.unblockUpdates();}}/** Returns true if we were able to drop buffered updates and return to the ACTIVE state */public boolean dropBufferedUpdates() {- versionInfo.blockUpdates();+ updateLocks.blockUpdates();try {if (state != State.BUFFERING) return false;@@ -1924,7 +1929,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {state = State.ACTIVE;} finally {- versionInfo.unblockUpdates();+ updateLocks.unblockUpdates();}return true;}@@ -1946,7 +1951,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {// block all updates to eliminate race conditions// reading state and acting on it in the update processor- versionInfo.blockUpdates();+ updateLocks.blockUpdates();try {cancelApplyBufferUpdate = false;if (state != State.BUFFERING) return null;@@ -1962,7 +1967,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {state = State.APPLYING_BUFFERED;} finally {- versionInfo.unblockUpdates();+ updateLocks.unblockUpdates();}if (ExecutorUtil.isShutdown(recoveryExecutor)) {@@ -2048,7 +2053,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {// change the state while updates are still blocked to prevent racesstate = State.ACTIVE;if (finishing) {- versionInfo.unblockUpdates();+ updateLocks.unblockUpdates();}// clean up in case we hit some unexpected exception and didn't get@@ -2100,7 +2105,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {return proc;});- OrderedExecutor executor =+ OrderedExecutor<BytesRef> executor =inSortedOrder ? null : req.getCoreContainer().getReplayUpdatesExecutor();AtomicInteger pendingTasks = new AtomicInteger(0);AtomicReference<SolrException> exceptionOnExecuteUpdate = new AtomicReference<>();@@ -2147,7 +2152,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {// after we've finished this recovery.// NOTE: our own updates won't be blocked since the thread holding a write lock can// lock a read lock.- versionInfo.blockUpdates();+ updateLocks.blockUpdates();finishing = true;o = tlogReader.next();} else {@@ -2309,34 +2314,27 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {}}- private Integer getBucketHash(UpdateCommand cmd) {+ private BytesRef getIndexedId(UpdateCommand cmd) {if (cmd instanceof AddUpdateCommand) {- BytesRef idBytes = ((AddUpdateCommand) cmd).getIndexedId();- if (idBytes == null) return null;- return DistributedUpdateProcessor.bucketHash(idBytes);+ return ((AddUpdateCommand) cmd).getIndexedId();}-if (cmd instanceof DeleteUpdateCommand) {- BytesRef idBytes = ((DeleteUpdateCommand) cmd).getIndexedId();- if (idBytes == null) return null;- return DistributedUpdateProcessor.bucketHash(idBytes);+ return ((DeleteUpdateCommand) cmd).getIndexedId();}-return null;}private void execute(UpdateCommand cmd,- OrderedExecutor executor,+ OrderedExecutor<BytesRef> executor,AtomicInteger pendingTasks,ThreadLocal<UpdateRequestProcessor> procTl,AtomicReference<SolrException> exceptionHolder) {assert cmd instanceof AddUpdateCommand || cmd instanceof DeleteUpdateCommand;if (executor != null) {- // by using the same hash as DUP, independent updates can avoid waiting for same bucketexecutor.execute(- getBucketHash(cmd),+ getIndexedId(cmd),() -> {try {// fail fastdiff --git a/solr/core/src/java/org/apache/solr/update/VersionBucket.java b/solr/core/src/java/org/apache/solr/update/VersionBucket.javadeleted file mode 100644index 7724b3a2cb4..00000000000--- a/solr/core/src/java/org/apache/solr/update/VersionBucket.java+++ /dev/null@@ -1,64 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.solr.update;--import java.io.IOException;-import java.util.concurrent.TimeUnit;--// TODO: make inner?-// TODO: store the highest possible in the index on a commit (but how to not block adds?)-// TODO: could also store highest possible in the transaction log after a commit.-// Or on a new index, just scan "version" for the max?-/**- * The default implementation which uses the intrinsic object monitor. It uses less memory but- * ignores the <code>lockTimeoutMs</code>.- *- * @lucene.internal- */-public class VersionBucket {-- @FunctionalInterface- public interface CheckedFunction<T, R> {- R apply() throws IOException;- }-- /** This will run the function with the intrinsic object monitor. */- public <T, R> R runWithLock(int lockTimeoutMs, CheckedFunction<T, R> function)- throws IOException {- synchronized (this) {- return function.apply();- }- }-- /** Nothing to do for the intrinsic object monitor. */- public void unlock() {}-- public void signalAll() {- notifyAll();- }-- public void awaitNanos(long nanosTimeout) {- try {- long millis = TimeUnit.NANOSECONDS.toMillis(nanosTimeout);- if (millis > 0) {- wait(millis);- }- } catch (InterruptedException e) {- throw new RuntimeException(e);- }- }-}diff --git a/solr/core/src/java/org/apache/solr/update/VersionInfo.java b/solr/core/src/java/org/apache/solr/update/VersionInfo.javaindex 09a39397197..ad7c427c403 100644--- a/solr/core/src/java/org/apache/solr/update/VersionInfo.java+++ b/solr/core/src/java/org/apache/solr/update/VersionInfo.java@@ -20,11 +20,8 @@ import static org.apache.solr.common.params.CommonParams.VERSION_FIELD;import java.io.IOException;import java.util.Map;-import java.util.concurrent.locks.ReadWriteLock;-import java.util.concurrent.locks.ReentrantReadWriteLock;import org.apache.lucene.queries.function.FunctionValues;import org.apache.lucene.queries.function.ValueSource;-import org.apache.lucene.util.BitUtil;import org.apache.lucene.util.BytesRef;import org.apache.solr.common.SolrException;import org.apache.solr.common.util.SuppressForbidden;@@ -33,18 +30,16 @@ import org.apache.solr.schema.SchemaField;import org.apache.solr.search.SolrIndexSearcher;import org.apache.solr.util.RefCounted;+/**+ * Related to the {@code _version_} field, in connection with the {@link UpdateLog}.+ *+ * @lucene.internal+ */public class VersionInfo {- private static final String SYS_PROP_BUCKET_VERSION_LOCK_TIMEOUT_MS =- "bucketVersionLockTimeoutMs";private final UpdateLog ulog;- private final int numBuckets;- private volatile VersionBucket[] buckets;- private final Object bucketsSync = new Object();- private final SchemaField versionField;- final ReadWriteLock lock = new ReentrantReadWriteLock(true);- private final int versionBucketLockTimeoutMs;+ private final SchemaField versionField;/*** Gets and returns the {@link org.apache.solr.common.params.CommonParams#VERSION_FIELD} from the@@ -82,47 +77,16 @@ public class VersionInfo {return sf;}- public VersionInfo(UpdateLog ulog, int nBuckets) {+ public VersionInfo(UpdateLog ulog) {this.ulog = ulog;IndexSchema schema = ulog.uhandler.core.getLatestSchema();versionField = getAndCheckVersionField(schema);- versionBucketLockTimeoutMs =- ulog.uhandler- .core- .getSolrConfig()- .get("updateHandler")- .get("versionBucketLockTimeoutMs")- .intVal(- Integer.parseInt(System.getProperty(SYS_PROP_BUCKET_VERSION_LOCK_TIMEOUT_MS, "0")));- numBuckets = BitUtil.nextHighestPowerOfTwo(nBuckets);}- public int getVersionBucketLockTimeoutMs() {- return versionBucketLockTimeoutMs;- }-- public void reload() {}-public SchemaField getVersionField() {return versionField;}- public void lockForUpdate() {- lock.readLock().lock();- }-- public void unlockForUpdate() {- lock.readLock().unlock();- }-- public void blockUpdates() {- lock.writeLock().lock();- }-- public void unblockUpdates() {- lock.writeLock().unlock();- }-/*// todo: initialize... use current time to start?// a clock that increments by 1 for every operation makes it easier to detect missing@@ -172,42 +136,6 @@ public class VersionInfo {}}- public long getOldClock() {- synchronized (clockSync) {- return vclock;- }- }-- public void updateClock(long clock) {- synchronized (clockSync) {- vclock = Math.max(vclock, clock);- }- }-- public VersionBucket bucket(int hash) {- // If this is a user provided hash, it may be poor in the right-hand bits.- // Make sure high bits are moved down, since only the low bits will matter.- // int h = hash + (hash >>> 8) + (hash >>> 16) + (hash >>> 24);- // Assume good hash codes for now.- int slot = hash & (numBuckets - 1);- if (buckets == null) {- synchronized (bucketsSync) {- if (buckets == null) {- buckets = createVersionBuckets();- }- }- }- return buckets[slot];- }-- private VersionBucket[] createVersionBuckets() {- VersionBucket[] buckets = new VersionBucket[numBuckets];- for (int i = 0; i < buckets.length; i++) {- buckets[i] = versionBucketLockTimeoutMs > 0 ? new TimedVersionBucket() : new VersionBucket();- }- return buckets;- }-public Long lookupVersion(BytesRef idBytes) {return ulog.lookupVersion(idBytes);}diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.javaindex 8588a0a34de..d8c88ac5786 100644--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java@@ -25,6 +25,7 @@ import java.lang.invoke.MethodHandles;import java.util.List;import java.util.Set;import java.util.concurrent.TimeUnit;+import java.util.concurrent.locks.Condition;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.CharsRefBuilder;import org.apache.solr.client.solrj.SolrRequest;@@ -43,7 +44,6 @@ import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.params.ShardParams;import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.params.UpdateParams;-import org.apache.solr.common.util.Hash;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.TimeSource;import org.apache.solr.handler.component.RealTimeGetComponent;@@ -57,9 +57,9 @@ import org.apache.solr.update.SolrCmdDistributor;import org.apache.solr.update.SolrCmdDistributor.Node;import org.apache.solr.update.SolrCmdDistributor.SolrError;import org.apache.solr.update.UpdateCommand;+import org.apache.solr.update.UpdateLocks;import org.apache.solr.update.UpdateLog;import org.apache.solr.update.UpdateShardHandler;-import org.apache.solr.update.VersionBucket;import org.apache.solr.update.VersionInfo;import org.apache.solr.util.TestInjection;import org.apache.solr.util.TimeOut;@@ -129,7 +129,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {private final AtomicUpdateDocumentMerger docMerger;private final UpdateLog ulog;- @VisibleForTesting VersionInfo vinfo;+ private final VersionInfo vinfo;private final boolean versionsStored;private boolean returnVersions;@@ -202,6 +202,11 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {// this.rsp = reqInfo != null ? reqInfo.getRsp() : null;}+ @VisibleForTesting+ protected UpdateLocks getUpdateLocks() {+ return ulog.getLocks();+ }+/*** @return the replica type of the collection.*/@@ -277,11 +282,6 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {isIndexChanged = true;}- public static int bucketHash(BytesRef idBytes) {- assert idBytes != null;- return Hash.murmurhash3_x86_32(idBytes.bytes, idBytes.offset, idBytes.length, 0);- }-/*** @return whether or not to drop this cmd* @throws IOException If there is a low-level I/O error.@@ -305,10 +305,6 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {}}- // This is only the hash for the bucket, and must be based only on the uniqueKey (i.e. do not- // use a pluggable hash here)- int bucketHash = bucketHash(idBytes);-// at this point, there is an update we need to try and apply.// we may or may not be the leader.@@ -335,36 +331,41 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {leaderLogicWithVersionIntegrityCheck(isReplayOrPeersync, isLeader, versionOnUpdate);boolean forwardedFromCollection = cmd.getReq().getParams().get(DISTRIB_FROM_COLLECTION) != null;- VersionBucket bucket = vinfo.bucket(bucketHash);-long dependentVersionFound = -1;// if this is an in-place update, check and wait if we should be waiting for a previous update// (on which this update depends), before entering the synchronized blockif (!leaderLogic && cmd.isInPlaceUpdate()) {- dependentVersionFound =- waitForDependentUpdates(cmd, versionOnUpdate, isReplayOrPeersync, bucket);+ dependentVersionFound = waitForDependentUpdates(cmd, versionOnUpdate, isReplayOrPeersync);if (dependentVersionFound == -1) {// it means the document has been deleted by now at the leader. drop this updatereturn true;}}- vinfo.lockForUpdate();- try {- long finalVersionOnUpdate = versionOnUpdate;- return bucket.runWithLock(- vinfo.getVersionBucketLockTimeoutMs(),- () ->- doVersionAdd(+ final long finalVersionOnUpdate = versionOnUpdate;+ return getUpdateLocks()+ .runWithLock(+ cmd.getIndexedId(),+ (Condition condition) -> {+ // just in case anyone is waiting let them know that we have a new update+ // we obtain the version when synchronized and then do the add so we can ensure that+ // if version1 < version2 then version1 is actually added before version2.++ // even if we don't store the version field, synchronizing+ // will enable us to know what version happened first, and thus enable+ // realtime-get to work reliably.+ // TODO: if versions aren't stored, do we need to set on the cmd anyway for some+ // reason?+ // there may be other reasons in the future for a version on the commands+ condition.signalAll();++ return doVersionAdd(cmd,finalVersionOnUpdate,isReplayOrPeersync,leaderLogic,- forwardedFromCollection,- bucket));- } finally {- vinfo.unlockForUpdate();- }+ forwardedFromCollection);+ });}private boolean doVersionAdd(@@ -372,172 +373,157 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {long versionOnUpdate,boolean isReplayOrPeersync,boolean leaderLogic,- boolean forwardedFromCollection,- VersionBucket bucket)+ boolean forwardedFromCollection)throws IOException {- try {- BytesRef idBytes = cmd.getIndexedId();- bucket.signalAll();- // just in case anyone is waiting let them know that we have a new update- // we obtain the version when synchronized and then do the add so we can ensure that- // if version1 < version2 then version1 is actually added before version2.-- // even if we don't store the version field, synchronizing on the bucket- // will enable us to know what version happened first, and thus enable- // realtime-get to work reliably.- // TODO: if versions aren't stored, do we need to set on the cmd anyway for some reason?- // there may be other reasons in the future for a version on the commands-- if (versionsStored) {-- if (leaderLogic) {-- if (forwardedFromCollection && ulog.getState() == UpdateLog.State.ACTIVE) {- // forwarded from a collection but we are not buffering so strip original version and- // apply our own- // see SOLR-5308- if (log.isInfoEnabled()) {- log.info("Removing version field from doc: {}", cmd.getPrintableId());- }- cmd.solrDoc.remove(CommonParams.VERSION_FIELD);- versionOnUpdate = 0;- }+ BytesRef idBytes = cmd.getIndexedId();- getUpdatedDocument(cmd, versionOnUpdate);+ if (versionsStored) {- // leaders can also be in buffering state during "migrate" API call, see SOLR-5308- if (forwardedFromCollection- && ulog.getState() != UpdateLog.State.ACTIVE- && isReplayOrPeersync == false) {- // we're not in an active state, and this update isn't from a replay, so buffer it.- if (log.isInfoEnabled()) {- log.info(- "Leader logic applied but update log is buffering: {}", cmd.getPrintableId());- }- cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);- ulog.add(cmd);- return true;+ if (leaderLogic) {++ if (forwardedFromCollection && ulog.getState() == UpdateLog.State.ACTIVE) {+ // forwarded from a collection but we are not buffering so strip original version and+ // apply our own+ // see SOLR-5308+ if (log.isInfoEnabled()) {+ log.info("Removing version field from doc: {}", cmd.getPrintableId());}+ cmd.solrDoc.remove(CommonParams.VERSION_FIELD);+ versionOnUpdate = 0;+ }- if (versionOnUpdate != 0) {- Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());- long foundVersion = lastVersion == null ? -1 : lastVersion;- if (versionOnUpdate == foundVersion- || (versionOnUpdate < 0 && foundVersion < 0)- || (versionOnUpdate == 1 && foundVersion > 0)) {- // we're ok if versions match, or if both are negative (all missing docs are equal),- // or if cmd specified it must exist (versionOnUpdate==1) and it does.- } else {- if (cmd.getReq().getParams().getBool(CommonParams.FAIL_ON_VERSION_CONFLICTS, true)- == false) {- return true;- }+ getUpdatedDocument(cmd, versionOnUpdate);- throw new SolrException(- ErrorCode.CONFLICT,- "version conflict for "- + cmd.getPrintableId()- + " expected="- + versionOnUpdate- + " actual="- + foundVersion);- }+ // leaders can also be in buffering state during "migrate" API call, see SOLR-5308+ if (forwardedFromCollection+ && ulog.getState() != UpdateLog.State.ACTIVE+ && isReplayOrPeersync == false) {+ // we're not in an active state, and this update isn't from a replay, so buffer it.+ if (log.isInfoEnabled()) {+ log.info("Leader logic applied but update log is buffering: {}", cmd.getPrintableId());}+ cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);+ ulog.add(cmd);+ return true;+ }- long version = vinfo.getNewClock();- cmd.setVersion(version);- cmd.getSolrInputDocument().setField(CommonParams.VERSION_FIELD, version);- } else {- // The leader forwarded us this update.- cmd.setVersion(versionOnUpdate);+ if (versionOnUpdate != 0) {+ Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());+ long foundVersion = lastVersion == null ? -1 : lastVersion;+ if (versionOnUpdate == foundVersion+ || (versionOnUpdate < 0 && foundVersion < 0)+ || (versionOnUpdate == 1 && foundVersion > 0)) {+ // we're ok if versions match, or if both are negative (all missing docs are equal),+ // or if cmd specified it must exist (versionOnUpdate==1) and it does.+ } else {+ if (cmd.getReq().getParams().getBool(CommonParams.FAIL_ON_VERSION_CONFLICTS, true)+ == false) {+ return true;+ }- if (shouldBufferUpdate(cmd, isReplayOrPeersync, ulog.getState())) {- // we're not in an active state, and this update isn't from a replay, so buffer it.- cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);- ulog.add(cmd);- return true;+ throw new SolrException(+ ErrorCode.CONFLICT,+ "version conflict for "+ + cmd.getPrintableId()+ + " expected="+ + versionOnUpdate+ + " actual="+ + foundVersion);}+ }- if (cmd.isInPlaceUpdate()) {- long prev = cmd.prevVersion;- Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());- if (lastVersion == null || Math.abs(lastVersion) < prev) {- // this was checked for (in waitForDependentUpdates()) before entering the- // synchronized block. So we shouldn't be here, unless what must've happened is: by- // the time synchronization block was entered, the prev update was deleted by DBQ.- // Since now that update is not in index, the vinfo.lookupVersion() is possibly giving- // us a version from the deleted list (which might be older than the prev update!)- UpdateCommand fetchedFromLeader = fetchFullUpdateFromLeader(cmd, versionOnUpdate);-- if (fetchedFromLeader instanceof DeleteUpdateCommand) {- if (log.isInfoEnabled()) {- log.info(- "In-place update of {} failed to find valid lastVersion to apply to, and the document was deleted at the leader subsequently.",- idBytes.utf8ToString());- }- versionDelete((DeleteUpdateCommand) fetchedFromLeader);- return true;- } else {- assert fetchedFromLeader instanceof AddUpdateCommand;- // Newer document was fetched from the leader. Apply that document instead of this- // current in-place update.- if (log.isInfoEnabled()) {- log.info(- "In-place update of {} failed to find valid lastVersion to apply to, forced to fetch full doc from leader: {}",- idBytes.utf8ToString(),- fetchedFromLeader);- }- // Make this update to become a non-inplace update containing the full document- // obtained from the leader- cmd.solrDoc = ((AddUpdateCommand) fetchedFromLeader).solrDoc;- cmd.prevVersion = -1;- cmd.setVersion((long) cmd.solrDoc.getFieldValue(CommonParams.VERSION_FIELD));- assert cmd.isInPlaceUpdate() == false;+ long version = vinfo.getNewClock();+ cmd.setVersion(version);+ cmd.getSolrInputDocument().setField(CommonParams.VERSION_FIELD, version);+ } else {+ // The leader forwarded us this update.+ cmd.setVersion(versionOnUpdate);++ if (shouldBufferUpdate(cmd, isReplayOrPeersync, ulog.getState())) {+ // we're not in an active state, and this update isn't from a replay, so buffer it.+ cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);+ ulog.add(cmd);+ return true;+ }++ if (cmd.isInPlaceUpdate()) {+ long prev = cmd.prevVersion;+ Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());+ if (lastVersion == null || Math.abs(lastVersion) < prev) {+ // this was checked for (in waitForDependentUpdates()) before entering the+ // synchronized block. So we shouldn't be here, unless what must've happened is: by+ // the time synchronization block was entered, the prev update was deleted by DBQ.+ // Since now that update is not in index, the vinfo.lookupVersion() is possibly giving+ // us a version from the deleted list (which might be older than the prev update!)+ UpdateCommand fetchedFromLeader = fetchFullUpdateFromLeader(cmd, versionOnUpdate);++ if (fetchedFromLeader instanceof DeleteUpdateCommand) {+ if (log.isInfoEnabled()) {+ log.info(+ "In-place update of {} failed to find valid lastVersion to apply to, and the document was deleted at the leader subsequently.",+ idBytes.utf8ToString());}+ versionDelete((DeleteUpdateCommand) fetchedFromLeader);+ return true;} else {- if (Math.abs(lastVersion) > prev) {- // this means we got a newer full doc update and in that case it makes no sense to- // apply the older inplace update. Drop this update+ assert fetchedFromLeader instanceof AddUpdateCommand;+ // Newer document was fetched from the leader. Apply that document instead of this+ // current in-place update.+ if (log.isInfoEnabled()) {log.info(- "Update was applied on version: {}, but last version I have is: {}. Dropping current update",- prev,- lastVersion);- return true;- } else {- // We're good, we should apply this update.+ "In-place update of {} failed to find valid lastVersion to apply to, forced to fetch full doc from leader: {}",+ idBytes.utf8ToString(),+ fetchedFromLeader);}+ // Make this update to become a non-inplace update containing the full document+ // obtained from the leader+ cmd.solrDoc = ((AddUpdateCommand) fetchedFromLeader).solrDoc;+ cmd.prevVersion = -1;+ cmd.setVersion((long) cmd.solrDoc.getFieldValue(CommonParams.VERSION_FIELD));+ assert cmd.isInPlaceUpdate() == false;}} else {- // if we aren't the leader, then we need to check that updates were not re-ordered- // we need to check the specific version for this id.- Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());- if (lastVersion != null && Math.abs(lastVersion) >= versionOnUpdate) {- // This update is a repeat, or was reordered. We need to drop this update.- if (log.isDebugEnabled()) {- log.debug("Dropping add update due to version {}", idBytes.utf8ToString());- }+ if (Math.abs(lastVersion) > prev) {+ // this means we got a newer full doc update and in that case it makes no sense to+ // apply the older inplace update. Drop this update+ log.info(+ "Update was applied on version: {}, but last version I have is: {}. Dropping current update",+ prev,+ lastVersion);return true;+ } else {+ // We're good, we should apply this update.}}- if (!isSubShardLeader- && replicaType == Replica.Type.TLOG- && (cmd.getFlags() & UpdateCommand.REPLAY) == 0) {- cmd.setFlags(cmd.getFlags() | UpdateCommand.IGNORE_INDEXWRITER);+ } else {+ // if we aren't the leader, then we need to check that updates were not re-ordered+ // we need to check the specific version for this id.+ Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());+ if (lastVersion != null && Math.abs(lastVersion) >= versionOnUpdate) {+ // This update is a repeat, or was reordered. We need to drop this update.+ if (log.isDebugEnabled()) {+ log.debug("Dropping add update due to version {}", idBytes.utf8ToString());+ }+ return true;}}+ if (!isSubShardLeader+ && replicaType == Replica.Type.TLOG+ && (cmd.getFlags() & UpdateCommand.REPLAY) == 0) {+ cmd.setFlags(cmd.getFlags() | UpdateCommand.IGNORE_INDEXWRITER);+ }}+ }- SolrInputDocument clonedDoc = shouldCloneCmdDoc() ? cmd.solrDoc.deepCopy() : null;+ SolrInputDocument clonedDoc = shouldCloneCmdDoc() ? cmd.solrDoc.deepCopy() : null;- // TODO: possibly set checkDeleteByQueries as a flag on the command?- doLocalAdd(cmd);+ // TODO: possibly set checkDeleteByQueries as a flag on the command?+ doLocalAdd(cmd);- if (clonedDoc != null) {- cmd.solrDoc = clonedDoc;- }- } finally {- bucket.unlock();+ if (clonedDoc != null) {+ cmd.solrDoc = clonedDoc;}+return false;}@@ -573,22 +559,16 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {* has been indexed.*/private long waitForDependentUpdates(- AddUpdateCommand cmd, long versionOnUpdate, boolean isReplayOrPeersync, VersionBucket bucket)- throws IOException {- long lastFoundVersion = 0;+ AddUpdateCommand cmd, long versionOnUpdate, boolean isReplayOrPeersync) throws IOException {TimeOut waitTimeout = new TimeOut(5, TimeUnit.SECONDS, TimeSource.NANO_TIME);- vinfo.lockForUpdate();- try {- lastFoundVersion =- bucket.runWithLock(- vinfo.getVersionBucketLockTimeoutMs(),- () ->- doWaitForDependentUpdates(- cmd, versionOnUpdate, isReplayOrPeersync, bucket, waitTimeout));- } finally {- vinfo.unlockForUpdate();- }+ long lastFoundVersion =+ getUpdateLocks()+ .runWithLock(+ cmd.getIndexedId(),+ (Condition condition) ->+ doWaitForDependentUpdates(+ cmd, versionOnUpdate, isReplayOrPeersync, condition, waitTimeout));if (Math.abs(lastFoundVersion) > cmd.prevVersion) {// This must've been the case due to a higher version full update succeeding concurrently,@@ -652,35 +632,34 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {AddUpdateCommand cmd,long versionOnUpdate,boolean isReplayOrPeersync,- VersionBucket bucket,+ Condition condition,TimeOut waitTimeout) {- long lastFoundVersion;- try {- Long lookedUpVersion = vinfo.lookupVersion(cmd.getIndexedId());- lastFoundVersion = lookedUpVersion == null ? 0L : lookedUpVersion;+ Long lookedUpVersion = vinfo.lookupVersion(cmd.getIndexedId());+ long lastFoundVersion = lookedUpVersion == null ? 0L : lookedUpVersion;- if (Math.abs(lastFoundVersion) < cmd.prevVersion) {- if (log.isDebugEnabled()) {- log.debug(- "Re-ordered inplace update. version={}, prevVersion={}, lastVersion={}, replayOrPeerSync={}, id={}",- (cmd.getVersion() == 0 ? versionOnUpdate : cmd.getVersion()),- cmd.prevVersion,- lastFoundVersion,- isReplayOrPeersync,- cmd.getPrintableId());- }+ if (Math.abs(lastFoundVersion) < cmd.prevVersion) {+ if (log.isDebugEnabled()) {+ log.debug(+ "Re-ordered inplace update. version={}, prevVersion={}, lastVersion={}, replayOrPeerSync={}, id={}",+ (cmd.getVersion() == 0 ? versionOnUpdate : cmd.getVersion()),+ cmd.prevVersion,+ lastFoundVersion,+ isReplayOrPeersync,+ cmd.getPrintableId());}+ }- while (Math.abs(lastFoundVersion) < cmd.prevVersion && !waitTimeout.hasTimedOut()) {- long timeLeftInNanos = waitTimeout.timeLeft(TimeUnit.NANOSECONDS);- if (timeLeftInNanos > 0) { // 0 means: wait forever until notified, but we don't want that.- bucket.awaitNanos(timeLeftInNanos);+ while (Math.abs(lastFoundVersion) < cmd.prevVersion && !waitTimeout.hasTimedOut()) {+ long timeLeftInNanos = waitTimeout.timeLeft(TimeUnit.NANOSECONDS);+ if (timeLeftInNanos > 0) { // 0 means: wait forever until notified, but we don't want that.+ try {+ condition.await(timeLeftInNanos, TimeUnit.NANOSECONDS);+ } catch (InterruptedException e) {+ throw new RuntimeException(e);}- lookedUpVersion = vinfo.lookupVersion(cmd.getIndexedId());- lastFoundVersion = lookedUpVersion == null ? 0L : lookedUpVersion;}- } finally {- bucket.unlock();+ lookedUpVersion = vinfo.lookupVersion(cmd.getIndexedId());+ lastFoundVersion = lookedUpVersion == null ? 0L : lookedUpVersion;}return lastFoundVersion;}@@ -958,7 +937,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {boolean leaderLogic =leaderLogicWithVersionIntegrityCheck(isReplayOrPeersync, isLeader, versionOnUpdate);- vinfo.blockUpdates();+ getUpdateLocks().blockUpdates();try {doLocalDeleteByQuery(cmd, versionOnUpdate, isReplayOrPeersync, leaderLogic);@@ -968,7 +947,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {// IndexReader (so cache misses will see up-to-date data)} finally {- vinfo.unblockUpdates();+ getUpdateLocks().unblockUpdates();}}@@ -1040,10 +1019,6 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {return false;}- // This is only the hash for the bucket, and must be based only on the uniqueKey (i.e. do not- // use a pluggable hash here)- int bucketHash = bucketHash(idBytes);-// at this point, there is an update we need to try and apply.// we may or may not be the leader.@@ -1062,25 +1037,18 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {leaderLogicWithVersionIntegrityCheck(isReplayOrPeersync, isLeader, versionOnUpdate);boolean forwardedFromCollection = cmd.getReq().getParams().get(DISTRIB_FROM_COLLECTION) != null;- VersionBucket bucket = vinfo.bucket(bucketHash);-- vinfo.lockForUpdate();- try {- long finalVersionOnUpdate = versionOnUpdate;- return bucket.runWithLock(- vinfo.getVersionBucketLockTimeoutMs(),- () ->- doVersionDelete(- cmd,- finalVersionOnUpdate,- signedVersionOnUpdate,- isReplayOrPeersync,- leaderLogic,- forwardedFromCollection,- bucket));- } finally {- vinfo.unlockForUpdate();- }+ long finalVersionOnUpdate = versionOnUpdate;+ return getUpdateLocks()+ .runWithLock(+ cmd.getIndexedId(),+ (Condition unused) ->+ doVersionDelete(+ cmd,+ finalVersionOnUpdate,+ signedVersionOnUpdate,+ isReplayOrPeersync,+ leaderLogic,+ forwardedFromCollection));}private boolean doVersionDelete(@@ -1089,94 +1057,89 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {long signedVersionOnUpdate,boolean isReplayOrPeersync,boolean leaderLogic,- boolean forwardedFromCollection,- VersionBucket bucket)+ boolean forwardedFromCollection)throws IOException {- try {- BytesRef idBytes = cmd.getIndexedId();- if (versionsStored) {+ BytesRef idBytes = cmd.getIndexedId();+ if (versionsStored) {- if (leaderLogic) {+ if (leaderLogic) {- if (forwardedFromCollection && ulog.getState() == UpdateLog.State.ACTIVE) {- // forwarded from a collection but we are not buffering so strip original version and- // apply our own- // see SOLR-5308- if (log.isInfoEnabled()) {- log.info("Removing version field from doc: {}", cmd.getId());- }- versionOnUpdate = signedVersionOnUpdate = 0;+ if (forwardedFromCollection && ulog.getState() == UpdateLog.State.ACTIVE) {+ // forwarded from a collection but we are not buffering so strip original version and+ // apply our own+ // see SOLR-5308+ if (log.isInfoEnabled()) {+ log.info("Removing version field from doc: {}", cmd.getId());}+ versionOnUpdate = signedVersionOnUpdate = 0;+ }- // leaders can also be in buffering state during "migrate" API call, see SOLR-5308- if (forwardedFromCollection- && ulog.getState() != UpdateLog.State.ACTIVE- && !isReplayOrPeersync) {- // we're not in an active state, and this update isn't from a replay, so buffer it.- if (log.isInfoEnabled()) {- log.info("Leader logic applied but update log is buffering: {}", cmd.getId());- }- cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);- ulog.delete(cmd);- return true;+ // leaders can also be in buffering state during "migrate" API call, see SOLR-5308+ if (forwardedFromCollection+ && ulog.getState() != UpdateLog.State.ACTIVE+ && !isReplayOrPeersync) {+ // we're not in an active state, and this update isn't from a replay, so buffer it.+ if (log.isInfoEnabled()) {+ log.info("Leader logic applied but update log is buffering: {}", cmd.getId());}+ cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);+ ulog.delete(cmd);+ return true;+ }- if (signedVersionOnUpdate != 0) {- Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());- long foundVersion = lastVersion == null ? -1 : lastVersion;- if ((signedVersionOnUpdate == foundVersion)- || (signedVersionOnUpdate < 0 && foundVersion < 0)- || (signedVersionOnUpdate == 1 && foundVersion > 0)) {- // we're ok if versions match, or if both are negative (all missing docs are equal),- // or if cmd specified it must exist (versionOnUpdate==1) and it does.- } else {- throw new SolrException(- ErrorCode.CONFLICT,- "version conflict for "- + cmd.getId()- + " expected="- + signedVersionOnUpdate- + " actual="- + foundVersion);- }+ if (signedVersionOnUpdate != 0) {+ Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());+ long foundVersion = lastVersion == null ? -1 : lastVersion;+ if ((signedVersionOnUpdate == foundVersion)+ || (signedVersionOnUpdate < 0 && foundVersion < 0)+ || (signedVersionOnUpdate == 1 && foundVersion > 0)) {+ // we're ok if versions match, or if both are negative (all missing docs are equal),+ // or if cmd specified it must exist (versionOnUpdate==1) and it does.+ } else {+ throw new SolrException(+ ErrorCode.CONFLICT,+ "version conflict for "+ + cmd.getId()+ + " expected="+ + signedVersionOnUpdate+ + " actual="+ + foundVersion);}+ }- long version = vinfo.getNewClock();- cmd.setVersion(-version);- } else {- cmd.setVersion(-versionOnUpdate);+ long version = vinfo.getNewClock();+ cmd.setVersion(-version);+ } else {+ cmd.setVersion(-versionOnUpdate);- if (ulog.getState() != UpdateLog.State.ACTIVE && isReplayOrPeersync == false) {- // we're not in an active state, and this update isn't from a replay, so buffer it.- cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);- ulog.delete(cmd);- return true;- }+ if (ulog.getState() != UpdateLog.State.ACTIVE && isReplayOrPeersync == false) {+ // we're not in an active state, and this update isn't from a replay, so buffer it.+ cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING);+ ulog.delete(cmd);+ return true;+ }- // if we aren't the leader, then we need to check that updates were not re-ordered- // we need to check the specific version for this id.- Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());- if (lastVersion != null && Math.abs(lastVersion) >= versionOnUpdate) {- // This update is a repeat, or was reordered. We need to drop this update.- if (log.isDebugEnabled()) {- log.debug("Dropping delete update due to version {}", idBytes.utf8ToString());- }- return true;+ // if we aren't the leader, then we need to check that updates were not re-ordered+ // we need to check the specific version for this id.+ Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId());+ if (lastVersion != null && Math.abs(lastVersion) >= versionOnUpdate) {+ // This update is a repeat, or was reordered. We need to drop this update.+ if (log.isDebugEnabled()) {+ log.debug("Dropping delete update due to version {}", idBytes.utf8ToString());}+ return true;+ }- if (!isSubShardLeader- && replicaType == Replica.Type.TLOG- && (cmd.getFlags() & UpdateCommand.REPLAY) == 0) {- cmd.setFlags(cmd.getFlags() | UpdateCommand.IGNORE_INDEXWRITER);- }+ if (!isSubShardLeader+ && replicaType == Replica.Type.TLOG+ && (cmd.getFlags() & UpdateCommand.REPLAY) == 0) {+ cmd.setFlags(cmd.getFlags() | UpdateCommand.IGNORE_INDEXWRITER);}}-- doLocalDelete(cmd);- return false;- } finally {- bucket.unlock();}++ doLocalDelete(cmd);+ return false;}private static boolean leaderLogicWithVersionIntegrityCheck(@@ -1202,7 +1165,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {if (vinfo != null) {long commitVersion = vinfo.getNewClock();cmd.setVersion(commitVersion);- vinfo.lockForUpdate();+ getUpdateLocks().lockForUpdate();}try {@@ -1221,7 +1184,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {} finally {if (vinfo != null) {- vinfo.unlockForUpdate();+ getUpdateLocks().unlockForUpdate();}}}diff --git a/solr/core/src/java/org/apache/solr/util/OrderedExecutor.java b/solr/core/src/java/org/apache/solr/util/OrderedExecutor.javaindex 65d2491ed1e..3e3abb4e312 100644--- a/solr/core/src/java/org/apache/solr/util/OrderedExecutor.java+++ b/solr/core/src/java/org/apache/solr/util/OrderedExecutor.java@@ -25,9 +25,9 @@ import java.util.concurrent.RejectedExecutionException;import java.util.concurrent.Semaphore;import org.apache.solr.common.util.ExecutorUtil;-public class OrderedExecutor implements Executor {+public class OrderedExecutor<T> implements Executor {private final ExecutorService delegate;- private final SparseStripedLock<Integer> sparseStripedLock;+ private final SparseStripedLock<T> sparseStripedLock;public OrderedExecutor(int numThreads, ExecutorService delegate) {this.delegate = delegate;@@ -51,7 +51,7 @@ public class OrderedExecutor implements Executor {* @param command the runnable task* @throws RejectedExecutionException if this task cannot be accepted for execution*/- public void execute(Integer lockId, Runnable command) {+ public void execute(T lockId, Runnable command) {try {sparseStripedLock.add(lockId);} catch (InterruptedException e) {diff --git a/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xmlindex 0372460c2b2..115fea2368b 100644--- a/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml+++ b/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml@@ -30,7 +30,6 @@<updateHandler class="solr.DirectUpdateHandler2"><updateLog><str name="dir">${solr.ulog.dir:}</str>- <int name="numVersionBuckets">${solr.ulog.numVersionBuckets:65536}</int></updateLog></updateHandler>diff --git a/solr/core/src/test/org/apache/solr/update/processor/DistributedUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/DistributedUpdateProcessorTest.javaindex dc430437121..0014260001a 100644--- a/solr/core/src/test/org/apache/solr/update/processor/DistributedUpdateProcessorTest.java+++ b/solr/core/src/test/org/apache/solr/update/processor/DistributedUpdateProcessorTest.java@@ -18,8 +18,6 @@package org.apache.solr.update.processor;import static org.hamcrest.CoreMatchers.is;-import static org.mockito.ArgumentMatchers.anyInt;-import static org.mockito.Mockito.doReturn;import java.io.IOException;import java.util.ArrayList;@@ -39,25 +37,18 @@ import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.update.AddUpdateCommand;import org.apache.solr.update.DeleteUpdateCommand;import org.apache.solr.update.SolrCmdDistributor;-import org.apache.solr.update.TimedVersionBucket;+import org.apache.solr.update.UpdateLocks;import org.apache.solr.update.UpdateLog;-import org.apache.solr.update.VersionInfo;import org.junit.AfterClass;import org.junit.BeforeClass;-import org.junit.Rule;import org.junit.Test;-import org.mockito.Mockito;-import org.mockito.junit.MockitoJUnit;-import org.mockito.junit.MockitoRule;public class DistributedUpdateProcessorTest extends SolrTestCaseJ4 {- @Rule public MockitoRule rule = MockitoJUnit.rule();private static ExecutorService executor;@BeforeClasspublic static void beforeClass() throws Exception {- assumeWorkingMockito();executor = ExecutorUtil.newMDCAwareCachedThreadPool(getClassName());System.setProperty("enable.update.log", "true");initCore(@@ -66,10 +57,8 @@ public class DistributedUpdateProcessorTest extends SolrTestCaseJ4 {}@AfterClass- public static void AfterClass() {- if (null != executor) { // may not have been initialized due to lack of mockito- executor.shutdown();- }+ public static void afterClass() {+ ExecutorUtil.shutdownAndAwaitTermination(executor);System.clearProperty("enable.update.log");}@@ -164,36 +153,12 @@ public class DistributedUpdateProcessorTest extends SolrTestCaseJ4 {*/private int runCommands(int threads,- int versionBucketLockTimeoutMs,+ int docLockTimeoutMs,SolrQueryRequest req,Function<DistributedUpdateProcessor, Boolean> function)throws IOException {- try (DistributedUpdateProcessor processor =- new DistributedUpdateProcessor(req, null, null, null)) {- if (versionBucketLockTimeoutMs > 0) {- // use TimedVersionBucket with versionBucketLockTimeoutMs- VersionInfo vinfo = Mockito.spy(processor.vinfo);- processor.vinfo = vinfo;-- doReturn(- new TimedVersionBucket() {- /** simulate the case: it takes 5 seconds to add the doc */- @Override- protected boolean tryLock(int lockTimeoutMs) {- boolean locked = super.tryLock(versionBucketLockTimeoutMs);- if (locked) {- try {- Thread.sleep(5000);- } catch (InterruptedException e) {- throw new RuntimeException(e);- }- }- return locked;- }- })- .when(vinfo)- .bucket(anyInt());- }++ try (DistributedUpdateProcessor processor = newDurp(req, docLockTimeoutMs)) {CountDownLatch latch = new CountDownLatch(1);Collection<Future<Boolean>> futures = new ArrayList<>();for (int t = 0; t < threads; ++t) {@@ -220,4 +185,45 @@ public class DistributedUpdateProcessorTest extends SolrTestCaseJ4 {return succeeded;}}++ private static DistributedUpdateProcessor newDurp(SolrQueryRequest req, long lockTimeoutMs) {+ if (lockTimeoutMs <= 0) {+ // default+ return new DistributedUpdateProcessor(req, null, null);+ }+ // customize UpdateLocks with the provided timeout. And simulate docs taking longer to index+ final var sleepMs = lockTimeoutMs + 1000;+ assert sleepMs > lockTimeoutMs;+ var sleepUrp =+ new UpdateRequestProcessor(null) {+ @Override+ public void processAdd(AddUpdateCommand cmd) throws IOException {+ sleep();+ super.processAdd(cmd);+ }++ @Override+ public void processDelete(DeleteUpdateCommand cmd) throws IOException {+ sleep();+ super.processDelete(cmd);+ }++ private void sleep() {+ try {+ Thread.sleep(sleepMs);+ } catch (InterruptedException e) {+ throw new RuntimeException(e);+ }+ }+ };++ return new DistributedUpdateProcessor(req, null, null, sleepUrp) {+ UpdateLocks updateLocks = new UpdateLocks(lockTimeoutMs);++ @Override+ protected UpdateLocks getUpdateLocks() {+ return updateLocks;+ }+ };+ }}diff --git a/solr/core/src/test/org/apache/solr/util/OrderedExecutorTest.java b/solr/core/src/test/org/apache/solr/util/OrderedExecutorTest.javaindex 4b49baa6e01..41099cf9c9c 100644--- a/solr/core/src/test/org/apache/solr/util/OrderedExecutorTest.java+++ b/solr/core/src/test/org/apache/solr/util/OrderedExecutorTest.java@@ -39,8 +39,8 @@ public class OrderedExecutorTest extends SolrTestCase {@Testpublic void testExecutionInOrder() {- OrderedExecutor orderedExecutor =- new OrderedExecutor(10, ExecutorUtil.newMDCAwareCachedThreadPool("executeInOrderTest"));+ OrderedExecutor<Integer> orderedExecutor =+ new OrderedExecutor<>(10, ExecutorUtil.newMDCAwareCachedThreadPool("executeInOrderTest"));IntBox intBox = new IntBox();for (int i = 0; i < 100; i++) {orderedExecutor.execute(1, () -> intBox.value++);@@ -53,8 +53,8 @@ public class OrderedExecutorTest extends SolrTestCase {public void testLockWhenQueueIsFull() {final ExecutorService controlExecutor =ExecutorUtil.newMDCAwareCachedThreadPool("testLockWhenQueueIsFull_control");- final OrderedExecutor orderedExecutor =- new OrderedExecutor(+ final OrderedExecutor<Integer> orderedExecutor =+ new OrderedExecutor<>(10, ExecutorUtil.newMDCAwareCachedThreadPool("testLockWhenQueueIsFull_test"));try {@@ -111,8 +111,8 @@ public class OrderedExecutorTest extends SolrTestCase {final ExecutorService controlExecutor =ExecutorUtil.newMDCAwareCachedThreadPool("testRunInParallel_control");- final OrderedExecutor orderedExecutor =- new OrderedExecutor(+ final OrderedExecutor<Integer> orderedExecutor =+ new OrderedExecutor<>(parallelism, ExecutorUtil.newMDCAwareCachedThreadPool("testRunInParallel_test"));try {@@ -216,8 +216,8 @@ public class OrderedExecutorTest extends SolrTestCase {base.put(i, i);run.put(i, i);}- OrderedExecutor orderedExecutor =- new OrderedExecutor(10, ExecutorUtil.newMDCAwareCachedThreadPool("testStress"));+ OrderedExecutor<Integer> orderedExecutor =+ new OrderedExecutor<>(10, ExecutorUtil.newMDCAwareCachedThreadPool("testStress"));for (int i = 0; i < 1000; i++) {int key = random().nextInt(N);base.put(key, base.get(key) + 1);@@ -233,8 +233,8 @@ public class OrderedExecutorTest extends SolrTestCase {@Testpublic void testMaxSize() throws InterruptedException {- OrderedExecutor orderedExecutor =- new OrderedExecutor(1, ExecutorUtil.newMDCAwareCachedThreadPool("single"));+ OrderedExecutor<Integer> orderedExecutor =+ new OrderedExecutor<>(1, ExecutorUtil.newMDCAwareCachedThreadPool("single"));CountDownLatch isRunning = new CountDownLatch(1);CountDownLatch blockingLatch = new CountDownLatch(1);diff --git a/solr/prometheus-exporter/src/test-files/solr/collection1/conf/solrconfig.xml b/solr/prometheus-exporter/src/test-files/solr/collection1/conf/solrconfig.xmlindex bbc9b68628b..6ca22087fe4 100644--- a/solr/prometheus-exporter/src/test-files/solr/collection1/conf/solrconfig.xml+++ b/solr/prometheus-exporter/src/test-files/solr/collection1/conf/solrconfig.xml@@ -35,7 +35,6 @@<updateLog><str name="dir">${solr.ulog.dir:}</str>- <int name="numVersionBuckets">${solr.ulog.numVersionBuckets:65536}</int></updateLog><autoCommit>diff --git a/solr/server/solr/configsets/_default/conf/solrconfig.xml b/solr/server/solr/configsets/_default/conf/solrconfig.xmlindex 92896c6d3f7..825993191ae 100644--- a/solr/server/solr/configsets/_default/conf/solrconfig.xml+++ b/solr/server/solr/configsets/_default/conf/solrconfig.xml@@ -256,16 +256,9 @@is recommended (see below)."dir" - the target directory for transaction logs, defaults to thesolr data directory.- "numVersionBuckets" - sets the number of buckets used to keep- track of max version values when checking for re-ordered- updates; increase this value to reduce the cost of- synchronizing access to version buckets during high-volume- indexing, this requires 8 bytes (long) * numVersionBuckets- of heap space per Solr core.--><updateLog><str name="dir">${solr.ulog.dir:}</str>- <int name="numVersionBuckets">${solr.ulog.numVersionBuckets:65536}</int></updateLog><!-- AutoCommitdiff --git a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xmlindex 1cdef7ca55f..c8250755dc7 100644--- a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml+++ b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml@@ -268,16 +268,9 @@is recommended (see below)."dir" - the target directory for transaction logs, defaults to thesolr data directory.- "numVersionBuckets" - sets the number of buckets used to keep- track of max version values when checking for re-ordered- updates; increase this value to reduce the cost of- synchronizing access to version buckets during high-volume- indexing, this requires 8 bytes (long) * numVersionBuckets- of heap space per Solr core.--><updateLog><str name="dir">${solr.ulog.dir:}</str>- <int name="numVersionBuckets">${solr.ulog.numVersionBuckets:65536}</int></updateLog><!-- AutoCommitdiff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adocindex 424591309de..812bbcbc3a8 100644--- a/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adoc+++ b/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adoc@@ -268,17 +268,6 @@ The minimum number of update records to keep across all the transaction log file+The maximum number of transaction log files to keep.-`numVersionBuckets`::-+-[%autowidth,frame=none]-|===-|Optional |Default: `65336`-|===-+-The number of buckets used to keep track of maximum version values when checking for re-ordered updates.-Increase this value to reduce the cost of synchronizing access to version buckets during high-volume indexing.-This requires `(8 bytes (long) * numVersionBuckets)` of heap space per Solr core.-`syncLevel`::+[%autowidth,frame=none]@@ -302,7 +291,6 @@ An example, to be included under `<updateHandler>` in `solrconfig.xml`, employin<str name="dir">${solr.ulog.dir:}</str><int name="numRecordsToKeep">500</int><int name="maxNumLogsToKeep">20</int>- <int name="numVersionBuckets">65536</int><str name="syncLevel">FSYNC</str></updateLog>----@@ -314,25 +302,3 @@ These can be triggered to occur after any commit (`event="postCommit"`) or onlyUsers can write custom update event listener classes in Solr plugins.As of Solr 7.1, `RunExecutableListener` was removed for security reasons.--== Other <updateHandler> Options--In some cases complex updates (such as spatial/shape) may take very long time to complete.-In the default configuration other updates that fall into the same internal version bucket will wait indefinitely.-Eventually these outstanding requests may pile up and lead to thread exhaustion and possibly to OutOfMemory errors.--The parameter `versionBucketLockTimeoutMs` helps to prevent that by specifying a timeout for long-running update requests.-If this limit is reached the update will fail but it won't block forever all other updates.--There is a memory cost associated with this setting.-Values greater than the default of `0` (unlimited timeout) cause Solr to use a different internal implementation of the version bucket, which increases memory consumption from ~1.5MB to ~6.8MB per Solr core.--An example of specifying this option under `<config>` section of `solrconfig.xml`:--[source,xml]------<updateHandler class="solr.DirectUpdateHandler2">- ...- <int name="versionBucketLockTimeoutMs">10000</int>-</updateHandler>-----diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adocindex e6413f8cd15..fed3fc7fd1b 100644--- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc+++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc@@ -67,6 +67,10 @@ It is always strongly recommended that you fully reindex your documents after aIn Solr 8, it was possible to add docValues to a schema without re-indexing via `UninvertDocValuesMergePolicy`, an advanced/expert utility.Due to changes in Lucene 9, that isn't possible any more.+== Solr 9.8+=== Configuration+In solrconfig.xml, the `numVersionBuckets` and `versionBucketLockTimeoutMs` settings are now obsolete and ignored; a warning will be logged if specified.+== Solr 9.7=== SchemaVersion upgraded to 1.7The default schemaVersion is now 1.7.