Apache Solr open-source search software
SOLR-18083: Fix replication and other general operations in readOnly mode (#4081)
Houston Putmanf9bdca8d0390
diff --git a/changelog/unreleased/solr-18083-fix-read-only-behavior.yml b/changelog/unreleased/solr-18083-fix-read-only-behavior.ymlnew file mode 100644index 00000000000..2aae02026c5--- /dev/null+++ b/changelog/unreleased/solr-18083-fix-read-only-behavior.yml@@ -0,0 +1,9 @@+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc+title: Fix operational issues with readOnly collections, such as restarting SolrNodes and replicating from the leader.+type: fixed # added, changed, fixed, deprecated, removed, dependency_update, security, other+authors:+ - name: Houston Putman+ nick: HoustonPutman+links:+ - name: SOLR-18083+ url: https://issues.apache.org/jira/browse/SOLR-18083diff --git a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.javaindex 0b8c2a744da..bf023d3cb09 100644--- a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java+++ b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java@@ -303,6 +303,8 @@ public class RecoveryStrategy implements Runnable, Closeable {// ureq.getParams().set(UpdateParams.OPEN_SEARCHER, onlyLeaderIndexes);// Why do we need to open searcher if "onlyLeaderIndexes"?ureq.getParams().set(UpdateParams.OPEN_SEARCHER, false);+ // If the leader is readOnly, do not fail since the core is already committed.+ ureq.getParams().set(UpdateParams.FAIL_ON_READ_ONLY, false);ureq.setAction(AbstractUpdateRequest.ACTION.COMMIT, false, true).process(client);}}@@ -657,15 +659,17 @@ public class RecoveryStrategy implements Runnable, Closeable {break;}- // we wait a bit so that any updates on the leader- // that started before they saw recovering state- // are sure to have finished (see SOLR-7141 for- // discussion around current value)- // TODO since SOLR-11216, we probably won't need this- try {- Thread.sleep(waitForUpdatesWithStaleStatePauseMilliSeconds);- } catch (InterruptedException e) {- Thread.currentThread().interrupt();+ if (!core.readOnly) {+ // we wait a bit so that any updates on the leader+ // that started before they saw recovering state+ // are sure to have finished (see SOLR-7141 for+ // discussion around current value)+ // TODO since SOLR-11216, we probably won't need this+ try {+ Thread.sleep(waitForUpdatesWithStaleStatePauseMilliSeconds);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ }}// first thing we just try to syncdiff --git a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.javaindex 16a29f89a58..96401345503 100644--- a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java+++ b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContext.java@@ -195,7 +195,7 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {// first cancel any current recoverycore.getUpdateHandler().getSolrCoreState().cancelRecovery();- if (weAreReplacement) {+ if (weAreReplacement && !core.readOnly) {// wait a moment for any floating updates to finishtry {Thread.sleep(2500);diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.javaindex 24c7dc83b0c..5e246e69ff1 100644--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java@@ -2476,7 +2476,7 @@ public class SolrCore implements SolrInfoBean, Closeable {true,directoryFactory);} else {- RefCounted<IndexWriter> writer = getSolrCoreState().getIndexWriter(this);+ RefCounted<IndexWriter> writer = getSolrCoreState().getIndexWriter(this, true);DirectoryReader newReader = null;try {newReader = indexReaderFactory.newReader(writer.get(), this);diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.javaindex 70480602807..7e44992da16 100644--- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java+++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java@@ -530,14 +530,14 @@ public class IndexFetcher {// we just clear ours and commitlog.info("New index in Leader. Deleting mine...");RefCounted<IndexWriter> iw =- solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(solrCore);+ solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(solrCore, true);try {iw.get().deleteAll();} finally {iw.decref();}assert TestInjection.injectDelayBeforeFollowerCommitRefresh();- if (skipCommitOnLeaderVersionZero) {+ if (skipCommitOnLeaderVersionZero || solrCore.readOnly) {openNewSearcherAndUpdateCommitPoint();} else {SolrQueryRequest req = new LocalSolrQueryRequest(solrCore, new ModifiableSolrParams());@@ -624,7 +624,7 @@ public class IndexFetcher {// are successfully deletedsolrCore.getUpdateHandler().newIndexWriter(true);RefCounted<IndexWriter> writer =- solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(null);+ solrCore.getUpdateHandler().getSolrCoreState().getIndexWriter(null, true);try {IndexWriter indexWriter = writer.get();int c = 0;diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.javaindex cdc5de9313b..7f50064ffe1 100644--- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java@@ -1378,7 +1378,7 @@ public class ReplicationHandler extends RequestHandlerBase// ensure the writer is initialized so that we have a list of commit pointsRefCounted<IndexWriter> iw =- core.getUpdateHandler().getSolrCoreState().getIndexWriter(core);+ core.getUpdateHandler().getSolrCoreState().getIndexWriter(core, true);iw.decref();} catch (IOException e) {diff --git a/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java b/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.javaindex 7b5d791ae42..7a04edad33e 100644--- a/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java+++ b/solr/core/src/java/org/apache/solr/handler/RequestHandlerUtils.java@@ -99,6 +99,7 @@ public class RequestHandlerUtils {cmd.maxOptimizeSegments =params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments);cmd.prepareCommit = params.getBool(UpdateParams.PREPARE_COMMIT, cmd.prepareCommit);+ cmd.failOnReadOnly = params.getBool(UpdateParams.FAIL_ON_READ_ONLY, cmd.failOnReadOnly);}/**diff --git a/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java b/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.javaindex 6a749af2ac9..2e4d0be5fb2 100644--- a/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java+++ b/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java@@ -39,6 +39,7 @@ public class CommitUpdateCommand extends UpdateCommand {public boolean expungeDeletes = false;public boolean softCommit = false;public boolean prepareCommit = false;+ public boolean failOnReadOnly = true; // fail the commit if the core or collection is readOnly/*** User provided commit data. Can be let to null if there is none. It is possible to commit this@@ -98,6 +99,9 @@ public class CommitUpdateCommand extends UpdateCommand {.append(softCommit).append(",prepareCommit=").append(prepareCommit);+ if (!failOnReadOnly) {+ sb.append(",failOnReadOnly=").append(failOnReadOnly);+ }if (commitData != null) {sb.append(",commitData=").append(commitData);}diff --git a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.javaindex 767c1ad0b77..4feb43abd9d 100644--- a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java+++ b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java@@ -106,8 +106,9 @@ public final class DefaultSolrCoreState extends SolrCoreState}@Override- public RefCounted<IndexWriter> getIndexWriter(SolrCore core) throws IOException {- if (core != null && (!core.indexEnabled || core.readOnly)) {+ public RefCounted<IndexWriter> getIndexWriter(SolrCore core, boolean readOnlyCompatible)+ throws IOException {+ if (core != null && (!core.indexEnabled || (!readOnlyCompatible && core.readOnly))) {throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Indexing is temporarily disabled");}diff --git a/solr/core/src/java/org/apache/solr/update/SolrCoreState.java b/solr/core/src/java/org/apache/solr/update/SolrCoreState.javaindex 4d6f02d229f..5016f9fdbb6 100644--- a/solr/core/src/java/org/apache/solr/update/SolrCoreState.java+++ b/solr/core/src/java/org/apache/solr/update/SolrCoreState.java@@ -178,7 +178,19 @@ public abstract class SolrCoreState {** @throws IOException If there is a low-level I/O error.*/- public abstract RefCounted<IndexWriter> getIndexWriter(SolrCore core) throws IOException;+ public RefCounted<IndexWriter> getIndexWriter(SolrCore core) throws IOException {+ return getIndexWriter(core, false);+ }++ /**+ * Get the current IndexWriter. If a new IndexWriter must be created, use the settings from the+ * given {@link SolrCore}. Be very careful using the {@code readOnlyCompatible} flag, by default+ * it should be false if the returned indexWriter will be used for writing.+ *+ * @throws IOException If there is a low-level I/O error.+ */+ public abstract RefCounted<IndexWriter> getIndexWriter(SolrCore core, boolean readOnlyCompatible)+ throws IOException;/*** Rollback the current IndexWriter. When creating the new IndexWriter use the settings from thediff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.javaindex 82424a5f06a..fe6f44501e7 100644--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java@@ -127,7 +127,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {protected final SolrQueryResponse rsp;private final AtomicUpdateDocumentMerger docMerger;- private final UpdateLog ulog;+ protected final UpdateLog ulog;private final VersionInfo vinfo;private final boolean versionsStored;private boolean returnVersions;diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.javaindex 34532421bf5..12c2695a29d 100644--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java@@ -161,7 +161,15 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {assert TestInjection.injectFailUpdateRequests();if (isReadOnly()) {- throw new SolrException(ErrorCode.FORBIDDEN, "Collection " + collection + " is read-only.");+ if (cmd.failOnReadOnly) {+ throw new SolrException(ErrorCode.FORBIDDEN, "Collection " + collection + " is read-only.");+ } else {+ // Committing on a readOnly core/collection is a no-op, since the core was committed when+ // becoming read-only and hasn't had any updates since.+ assert ulog == null || !ulog.hasUncommittedChanges()+ : "Uncommitted changes found when trying to commit on a read-only collection";+ return;+ }}List<SolrCmdDistributor.Node> nodes = null;diff --git a/solr/solrj/src/java/org/apache/solr/common/params/UpdateParams.java b/solr/solrj/src/java/org/apache/solr/common/params/UpdateParams.javaindex f14252fb970..df6b11f1f3a 100644--- a/solr/solrj/src/java/org/apache/solr/common/params/UpdateParams.java+++ b/solr/solrj/src/java/org/apache/solr/common/params/UpdateParams.java@@ -46,6 +46,9 @@ public interface UpdateParams {/** expert: calls IndexWriter.prepareCommit */public static String PREPARE_COMMIT = "prepareCommit";+ /** Fail a commit when the core or collection is in read-only mode */+ public static String FAIL_ON_READ_ONLY = "failOnReadOnly";+/** Rollback update commands */public static String ROLLBACK = "rollback";