Apache Solr open-source search software
SOLR-17942: Raising configurable RAM per thread hard limit using VarHandle (#3741)
Puneet Ahuja6f2044cb7102
diff --git a/changelog/unreleased/puneet/SOLR-17942-raising-ramPerThreadHardLimit.yml b/changelog/unreleased/puneet/SOLR-17942-raising-ramPerThreadHardLimit.ymlnew file mode 100644index 00000000000..e503e37c697--- /dev/null+++ b/changelog/unreleased/puneet/SOLR-17942-raising-ramPerThreadHardLimit.yml@@ -0,0 +1,8 @@+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc+title: Raising the per ramPerThreadHardLimit to be configurable >2GB+type: other # added, changed, fixed, deprecated, removed, dependency_update, security, other+authors:+ - name: Puneet Ahuja+links:+ - name: SOLR-17942+ url: https://issues.apache.org/jira/browse/SOLR-17942diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.javaindex 236c9eb72bf..19a8e0135b9 100644--- a/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java+++ b/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java@@ -20,6 +20,7 @@ import static org.apache.solr.core.XmlConfigFile.assertWarnOrFail;import java.io.IOException;import java.lang.invoke.MethodHandles;+import java.lang.invoke.VarHandle;import java.util.Collections;import java.util.Map;import org.apache.lucene.analysis.Analyzer;@@ -34,6 +35,7 @@ import org.apache.lucene.util.InfoStream;import org.apache.solr.common.ConfigNode;import org.apache.solr.common.MapSerializable;import org.apache.solr.common.util.NamedList;+import org.apache.solr.common.util.SuppressForbidden;import org.apache.solr.core.DirectoryFactory;import org.apache.solr.core.PluginInfo;import org.apache.solr.core.SolrConfig;@@ -246,7 +248,11 @@ public class SolrIndexConfig implements MapSerializable {if (ramBufferSizeMB != -1) iwc.setRAMBufferSizeMB(ramBufferSizeMB);if (ramPerThreadHardLimitMB != -1) {- iwc.setRAMPerThreadHardLimitMB(ramPerThreadHardLimitMB);+ if (ramPerThreadHardLimitMB > 2048) {+ setPerThreadRAMLimitViaVarHandle(iwc, ramPerThreadHardLimitMB);+ } else {+ iwc.setRAMPerThreadHardLimitMB(ramPerThreadHardLimitMB);+ }}if (maxCommitMergeWaitMillis > 0) {@@ -346,4 +352,33 @@ public class SolrIndexConfig implements MapSerializable {return scheduler;}++ @SuppressForbidden(reason = "Need to override Lucene's 2GB per-thread limit for large datasets")+ private static void setPerThreadRAMLimitViaVarHandle(IndexWriterConfig config, int limitMB) {+ MethodHandles.Lookup lookup = MethodHandles.lookup();+ VarHandle fieldHandle = null;+ Class<?> currentClass = config.getClass();++ // Traverse the hierarchy to find the field+ while (currentClass != null && fieldHandle == null) {+ try {+ MethodHandles.Lookup privateLookup = MethodHandles.privateLookupIn(currentClass, lookup);+ fieldHandle = privateLookup.findVarHandle(currentClass, "perThreadHardLimitMB", int.class);+ } catch (IllegalAccessException | NoSuchFieldException e) {+ currentClass = currentClass.getSuperclass();+ }+ }++ if (fieldHandle == null) {+ log.error("Could not find VarHandle for perThreadHardLimitMB field");+ return;+ }++ try {+ fieldHandle.set(config, limitMB);+ log.info("Set perThreadHardLimitMB to {} MB via VarHandle", limitMB);+ } catch (RuntimeException | Error e) {+ log.error("Failed to set per-thread RAM limit via VarHandle", e);+ }+ }}diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adocindex 3ffa49ef435..aa6a4985323 100644--- a/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc+++ b/solr/solr-ref-guide/modules/configuration-guide/pages/index-segments-merging.adoc@@ -77,7 +77,7 @@ The default is `false`.=== ramPerThreadHardLimitMBSets the maximum memory (defined in megabytes) consumption per thread triggering a forced flush if exceeded.-The given value must be greater than `0` and less than `2048` MB (2GB).+The given value must be greater than `0`.[source,xml]----@@ -86,6 +86,8 @@ The given value must be greater than `0` and less than `2048` MB (2GB).NOTE: This is an expert level parameter as it triggers forced flush even if `ramBufferSizeMB` has not been exceeded.+WARNING: Values greater than 2048 MB (2GB) will bypass Lucene's built-in limit using reflection. This is an advanced feature that should only be used by experienced administrators who understand the memory implications.+== Merging Index SegmentsThe following settings define when segments are merged.