Apache Solr open-source search software
SOLR-17984: Create a Solr MergePolicyFactory for MergeOnFlushMergePolicy (#3848)
Houston Putman616c7d670d42
diff --git a/changelog/unreleased/SOLR-17984-merge-on-flush-policy.yml b/changelog/unreleased/SOLR-17984-merge-on-flush-policy.ymlnew file mode 100644index 00000000000..a3e3a7e4f21--- /dev/null+++ b/changelog/unreleased/SOLR-17984-merge-on-flush-policy.yml@@ -0,0 +1,10 @@+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc+title: Enable MergeOnFlushMergePolicy in Solr+type: added # added, changed, fixed, deprecated, removed, dependency_update, security, other+authors:+ - name: Houston Putman+ nick: HoustonPutman+ url: https://home.apache.org/phonebook.html?uid=houston+links:+ - name: SOLR-17984+ url: https://issues.apache.org/jira/browse/SOLR-17984diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.javaindex 1c7ca0ee6b1..4d2812ca668 100644--- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java+++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java@@ -111,7 +111,8 @@ public class SolrResourceLoader"security.cert.","handler.sql.","crossdc.handler.",- "crossdc.update.processor."+ "crossdc.update.processor.",+ "index."};private static final Charset UTF_8 = StandardCharsets.UTF_8;public static final String SOLR_RESOURCELOADING_RESTRICTED_ENABLED_PARAM =diff --git a/solr/core/src/java/org/apache/solr/index/MergeOnFlushMergePolicyFactory.java b/solr/core/src/java/org/apache/solr/index/MergeOnFlushMergePolicyFactory.javanew file mode 100644index 00000000000..d03ddcd334a--- /dev/null+++ b/solr/core/src/java/org/apache/solr/index/MergeOnFlushMergePolicyFactory.java@@ -0,0 +1,50 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.index;++import org.apache.lucene.index.MergePolicy;+import org.apache.lucene.sandbox.index.MergeOnFlushMergePolicy;+import org.apache.solr.core.SolrResourceLoader;+import org.apache.solr.schema.IndexSchema;++/** A {@link MergePolicyFactory} for {@code SortingMergePolicy} objects. */+public class MergeOnFlushMergePolicyFactory extends WrapperMergePolicyFactory {++ private static final String SSTMB = "smallSegmentThresholdMB";++ protected final Double smallSegmentThresholdMB;++ public MergeOnFlushMergePolicyFactory(+ SolrResourceLoader resourceLoader, MergePolicyFactoryArgs args, IndexSchema schema) {+ super(resourceLoader, args, schema);+ final String smallSegmentThresholdMBArg = (String) args.remove(SSTMB);+ if (smallSegmentThresholdMBArg == null) {+ this.smallSegmentThresholdMB = null;+ } else {+ this.smallSegmentThresholdMB = Double.parseDouble(smallSegmentThresholdMBArg);+ }+ }++ @Override+ protected MergePolicy getMergePolicyInstance(MergePolicy wrappedMP) {+ final MergeOnFlushMergePolicy mp = new MergeOnFlushMergePolicy(wrappedMP);+ if (smallSegmentThresholdMB != null) {+ mp.setSmallSegmentThresholdMB(smallSegmentThresholdMB);+ }+ return mp;+ }+}diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergeonflushmergepolicyfactory.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergeonflushmergepolicyfactory.xmlnew file mode 100644index 00000000000..a798d7155d9--- /dev/null+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergeonflushmergepolicyfactory.xml@@ -0,0 +1,59 @@+<?xml version="1.0" ?>++<!--+ Licensed to the Apache Software Foundation (ASF) under one or more+ contributor license agreements. See the NOTICE file distributed with+ this work for additional information regarding copyright ownership.+ The ASF licenses this file to You under the Apache License, Version 2.0+ (the "License"); you may not use this file except in compliance with+ the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+-->++<config>+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.MockDirectoryFactory}"/>+ <schemaFactory class="ClassicIndexSchemaFactory"/>++ <indexConfig>+ <mergePolicyFactory class="org.apache.solr.index.SortingMergePolicyFactory">+ <str name="sort">${mergePolicySort:timestamp_i_dvo desc}</str>+ <str name="wrapped.prefix">in</str>+ <str name="in.class">solr.MergeOnFlushMergePolicyFactory</str>+ <str name="in.smallSegmentThresholdMB">10</str>+ <str name="in.wrapped.prefix">in</str>+ <str name="in.in.class">org.apache.solr.util.RandomForceMergePolicyFactory</str>+ </mergePolicyFactory>+ <lockType>${solr.tests.lockType:single}</lockType>+ </indexConfig>++ <requestHandler name="/select" class="solr.SearchHandler" />++ <updateHandler class="solr.DirectUpdateHandler2">+ <updateLog>+ <str name="dir">${solr.ulog.dir:}</str>+ </updateLog>++ <autoCommit>+ <maxTime>${solr.autoCommit.maxTime:-1}</maxTime>+ <openSearcher>false</openSearcher>+ </autoCommit>++ <autoSoftCommit>+ <maxTime>${solr.autoSoftCommit.maxTime:-1}</maxTime>+ </autoSoftCommit>+ </updateHandler>+ <initParams path="/select">+ <lst name="defaults">+ <str name="df">text</str>+ </lst>+ </initParams>++</config>diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.javaindex 066be8e04f5..79f936ddd2f 100644--- a/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java+++ b/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java@@ -25,6 +25,7 @@ import org.apache.lucene.index.MergePolicy;import org.apache.lucene.index.SimpleMergedSegmentWarmer;import org.apache.lucene.index.TieredMergePolicy;import org.apache.lucene.misc.index.BPReorderingMergePolicy;+import org.apache.lucene.sandbox.index.MergeOnFlushMergePolicy;import org.apache.lucene.search.Sort;import org.apache.lucene.search.SortField;import org.apache.solr.SolrTestCaseJ4;@@ -36,6 +37,7 @@ import org.apache.solr.core.TestMergePolicyConfig;import org.apache.solr.index.SortingMergePolicy;import org.apache.solr.schema.IndexSchema;import org.apache.solr.schema.IndexSchemaFactory;+import org.apache.solr.util.RandomForceMergePolicy;import org.junit.After;import org.junit.BeforeClass;import org.junit.Test;@@ -58,6 +60,8 @@ public class SolrIndexConfigTest extends SolrTestCaseJ4 {"solrconfig-sortingmergepolicyfactory.xml";private static final String solrConfigFileNameBPReorderingMergePolicyFactory ="solrconfig-bpreorderingmergepolicyfactory.xml";+ private static final String solrConfigFileNameMergeOnFlushMergePolicyFactory =+ "solrconfig-mergeonflushmergepolicyfactory.xml";private static final String schemaFileName = "schema.xml";private static boolean compoundMergePolicySort = false;@@ -172,6 +176,47 @@ public class SolrIndexConfigTest extends SolrTestCaseJ4 {assertEquals("SortingMergePolicy.getSort", expected, actual);}+ public void testMergeOnFlushMPSolrIndexConfigCreation() throws Exception {+ final SortField sortField1 = new SortField("timestamp_i_dvo", SortField.Type.INT, true);+ final SortField sortField2 = new SortField("id", SortField.Type.STRING, false);+ sortField2.setMissingValue(SortField.STRING_LAST);++ SolrConfig solrConfig =+ new SolrConfig(instanceDir, solrConfigFileNameMergeOnFlushMergePolicyFactory);+ SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null);+ assertNotNull(solrIndexConfig);+ IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema(schemaFileName, solrConfig);++ h.getCore().setLatestSchema(indexSchema);+ IndexWriterConfig iwc = solrIndexConfig.toIndexWriterConfig(h.getCore());++ final MergePolicy mergePolicy = iwc.getMergePolicy();+ assertNotNull("null mergePolicy", mergePolicy);+ assertTrue(+ "mergePolicy (" + mergePolicy + ") is not a SortingMergePolicy",+ mergePolicy instanceof SortingMergePolicy);+ final SortingMergePolicy sortingMergePolicy = (SortingMergePolicy) mergePolicy;++ MergePolicy firstInnerPolicy = sortingMergePolicy.unwrap();+ assertNotNull("null firstInnerMergePolicy", firstInnerPolicy);+ assertTrue(+ "mergePolicy (" + firstInnerPolicy + ") is not a MergeOnFlushMergePolicy",+ firstInnerPolicy instanceof MergeOnFlushMergePolicy);+ final MergeOnFlushMergePolicy mergeOnFlushMergePolicy =+ (MergeOnFlushMergePolicy) firstInnerPolicy;+ assertEquals(+ "Wrong maxSegmentThresholdMB for MergeOnFlushMergePolicy",+ 10,+ mergeOnFlushMergePolicy.getSmallSegmentThresholdMB(),+ .01);++ MergePolicy secondInnerPolicy = mergeOnFlushMergePolicy.unwrap();+ assertNotNull("null secondInnerMergePolicy", secondInnerPolicy);+ assertTrue(+ "mergePolicy (" + secondInnerPolicy + ") is not a RandomForceMergePolicyFactory",+ secondInnerPolicy instanceof RandomForceMergePolicy);+ }+public void testBPReorderingMPSolrIndexConfigCreation() throws Exception {SolrConfig solrConfig =new SolrConfig(instanceDir, solrConfigFileNameBPReorderingMergePolicyFactory);diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adocindex a06851de955..3ffa49ef435 100644--- a/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc+++ b/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc@@ -96,7 +96,7 @@ Defines how merging segments is done.The default in Solr is to use `TieredMergePolicy`, which merges segments of approximately equal size, subject to an allowed number of segments per tier.-Other policies available are the `LogByteSizeMergePolicy` and `LogDocMergePolicy`.+Other policies available are the `LogByteSizeMergePolicy`, `LogDocMergePolicy` and `MergeOnFlushMergePolicy`.For more information on these policies, please see {lucene-javadocs}/core/org/apache/lucene/index/MergePolicy.html[the MergePolicy javadocs].[source,xml]@@ -134,6 +134,9 @@ Having fewer segments in the index generally accelerates searches, because thereIt also can also result in fewer physical files on disk.But to keep the number of segments low, merges will occur more often, which can add load to the system and slow down updates to the index.+`MergeOnFlushMergePolicy` allows users to ensure that all segments are larger than a specified `smallSegmentThresholdMB` option (default `100`).+When using this merge policy, all segments smaller than the given threshold will be merged during a commit.+Conversely, keeping more segments can accelerate indexing, because merges happen less often, making an update is less likely to trigger a merge.But searches become more computationally expensive and will likely be slower, because search terms must be looked up in more index segments.Faster index updates also means shorter commit turnaround times, which means more timely search results.