Apache Solr open-source search software
SOLR-17725: Add MergePolicy to block older segments from participating in merges (#3883)
Rahul Goswami62da63f09d7e
diff --git a/changelog/unreleased/SOLR-17725-Merge-policy-to-upgrade-index-for-compatibility-with-future-Solr-version.yml b/changelog/unreleased/SOLR-17725-Merge-policy-to-upgrade-index-for-compatibility-with-future-Solr-version.ymlnew file mode 100644index 00000000000..502be026e8e--- /dev/null+++ b/changelog/unreleased/SOLR-17725-Merge-policy-to-upgrade-index-for-compatibility-with-future-Solr-version.yml@@ -0,0 +1,8 @@+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc+title: Merge policy to upgrade index for compatibility with future Solr version+type: added+authors:+ - name: Rahul Goswami+links:+ - name: SOLR-17725+ url: https://issues.apache.org/jira/browse/SOLR-17725diff --git a/solr/core/src/java/org/apache/solr/index/LatestVersionFilterMergePolicy.java b/solr/core/src/java/org/apache/solr/index/LatestVersionFilterMergePolicy.javanew file mode 100644index 00000000000..c5a088220b2--- /dev/null+++ b/solr/core/src/java/org/apache/solr/index/LatestVersionFilterMergePolicy.java@@ -0,0 +1,111 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.index;++import java.io.IOException;+import java.util.Map;+import org.apache.lucene.index.FilterMergePolicy;+import org.apache.lucene.index.MergePolicy;+import org.apache.lucene.index.MergeTrigger;+import org.apache.lucene.index.SegmentCommitInfo;+import org.apache.lucene.index.SegmentInfos;+import org.apache.lucene.util.Version;++/**+ * Prevents any older version segment (i.e. older than current lucene major version), either+ * original or one derived as a result of merging with an older version segment, from being+ * considered for merges. That way a snapshot of older segments remains consistent. This assists in+ * upgrading to a future Lucene major version if existing documents are reindexed in the current+ * version with this merge policy in place.+ */+public class LatestVersionFilterMergePolicy extends FilterMergePolicy {++ public LatestVersionFilterMergePolicy(MergePolicy in) {+ super(in);+ }++ @Override+ public MergeSpecification findMerges(+ MergeTrigger mergeTrigger, SegmentInfos infos, MergeContext mergeContext) throws IOException {+ return in.findMerges(mergeTrigger, getFilteredInfos(infos), mergeContext);+ }++ @Override+ public MergeSpecification findForcedMerges(+ SegmentInfos infos,+ int maxSegmentCount,+ Map<SegmentCommitInfo, Boolean> segmentsToMerge,+ MergeContext mergeContext)+ throws IOException {+ return in.findForcedMerges(+ getFilteredInfos(infos), maxSegmentCount, segmentsToMerge, mergeContext);+ }++ @Override+ public MergeSpecification findForcedDeletesMerges(SegmentInfos infos, MergeContext mergeContext)+ throws IOException {+ return in.findForcedDeletesMerges(getFilteredInfos(infos), mergeContext);+ }++ @Override+ public MergeSpecification findFullFlushMerges(+ MergeTrigger mergeTrigger, SegmentInfos infos, MergeContext mergeContext) throws IOException {+ return in.findFullFlushMerges(mergeTrigger, getFilteredInfos(infos), mergeContext);+ }++ private SegmentInfos getFilteredInfos(SegmentInfos infos) {+ SegmentInfos infosClone = null;++ for (SegmentCommitInfo info : infos) {+ if (!allowSegmentForMerge(info)) {+ // There are older version segments present.+ // We should not remove from the original SegmentInfos. Hence we clone.+ infosClone = infos.clone();+ infosClone.clear();+ break;+ }+ }++ if (infosClone == null) {+ // All segments are latest major version and allowed to participate in merge+ return infos;+ } else {+ // Either mixed versions or all older version segments.+ // If we are here, most runs should fall in the former case.+ // The latter case should only happen once right after an upgrade, so we are ok with incurring+ // this redundant iteration for that one time to keep the logic simple+ for (SegmentCommitInfo info : infos) {+ if (allowSegmentForMerge(info)) {+ infosClone.add(info);+ }+ }+ }++ return infosClone;+ }++ /**+ * Determines if a SegmentCommitInfo should be part of the candidate set of segments that will be+ * considered for merges. By default, we only allow LATEST version segments to participate in+ * merges.+ */+ protected boolean allowSegmentForMerge(SegmentCommitInfo info) {+ return info.info.getMinVersion() != null+ && info.info.getMinVersion().major == Version.LATEST.major;+ }+}diff --git a/solr/core/src/java/org/apache/solr/index/LatestVersionMergePolicyFactory.java b/solr/core/src/java/org/apache/solr/index/LatestVersionMergePolicyFactory.javanew file mode 100644index 00000000000..047830d3199--- /dev/null+++ b/solr/core/src/java/org/apache/solr/index/LatestVersionMergePolicyFactory.java@@ -0,0 +1,43 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.index;++import org.apache.lucene.index.MergePolicy;+import org.apache.lucene.index.TieredMergePolicy;+import org.apache.solr.core.SolrResourceLoader;+import org.apache.solr.schema.IndexSchema;++/**+ * A {@link MergePolicyFactory} for {@link LatestVersionFilterMergePolicy} objects. The returned+ * LatestVersionFilterMergePolicy instance blocks older version segments (< current version of+ * Lucene) from participating in merges and delegates the merging to a TieredMergePolicy instance by+ * default. This can be used to reindex the data and ensure all segments are the latest version+ * segments by the end of the reindexing. This can help prepare the index for upgrade to a later+ * version of Solr/Lucene even if it was initially created on a now unsupported version+ */+public class LatestVersionMergePolicyFactory extends SimpleMergePolicyFactory {++ public LatestVersionMergePolicyFactory(+ SolrResourceLoader resourceLoader, MergePolicyFactoryArgs args, IndexSchema schema) {+ super(resourceLoader, args, schema);+ }++ @Override+ protected MergePolicy getMergePolicyInstance() {+ return new LatestVersionFilterMergePolicy(new TieredMergePolicy());+ }+}