Apache Solr open-source search software
SOLR-16328 : intern() strings in DocCollection to reduce memory footp… (#965)
Noble Paul1b7ff717b191
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex f6f59a16594..9113045788c 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -92,6 +92,8 @@ Optimizations* SOLR-16336: avoid fetching solrconfig.xml & schema.xml for already cached schema and config (noble)+* SOLR-16328: intern() strings in DocCollection to reduce memory footprint (noble)+Bug Fixes---------------------* SOLR-15918: Skip repetitive parent znode creation on config set upload (Mike Drob)diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.javaindex a59b3c0ed81..5e5bda2ae1f 100644--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java@@ -29,6 +29,7 @@ import static org.apache.solr.common.params.CommonParams.ZK_STATUS_PATH;import static org.apache.solr.core.CorePropertiesLocator.PROPERTIES_FILENAME;import static org.apache.solr.security.AuthenticationPlugin.AUTHENTICATION_PLUGIN_PROP;+import com.github.benmanes.caffeine.cache.Interner;import com.google.common.annotations.VisibleForTesting;import com.google.common.collect.ImmutableMap;import com.google.common.collect.Maps;@@ -56,6 +57,7 @@ import java.util.concurrent.ExecutionException;import java.util.concurrent.ExecutorService;import java.util.concurrent.Future;import java.util.concurrent.TimeoutException;+import java.util.function.Function;import java.util.function.Supplier;import java.util.stream.Collectors;import org.apache.commons.lang3.StringUtils;@@ -89,6 +91,7 @@ import org.apache.solr.common.AlreadyClosedException;import org.apache.solr.common.SolrException;import org.apache.solr.common.SolrException.ErrorCode;import org.apache.solr.common.cloud.Aliases;+import org.apache.solr.common.cloud.ClusterState;import org.apache.solr.common.cloud.DocCollection;import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.cloud.Replica.State;@@ -150,6 +153,8 @@ import org.apache.solr.util.RefCounted;import org.apache.solr.util.StartupLoggingUtils;import org.apache.solr.util.stats.MetricUtils;import org.apache.zookeeper.KeeperException;+import org.noggit.JSONParser;+import org.noggit.ObjectBuilder;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -383,6 +388,7 @@ public class CoreContainer {if (null != this.cfg.getBooleanQueryMaxClauseCount()) {IndexSearcher.setMaxClauseCount(this.cfg.getBooleanQueryMaxClauseCount());}+ setWeakStringInterner();this.coresLocator = locator;this.containerProperties = new Properties(config.getSolrProperties());this.asyncSolrCoreLoad = asyncSolrCoreLoad;@@ -2424,6 +2430,34 @@ public class CoreContainer {public void runAsync(Runnable r) {coreContainerAsyncTaskExecutor.submit(r);}++ public static void setWeakStringInterner() {+ boolean enable = "true".equals(System.getProperty("solr.use.str.intern", "true"));+ if (!enable) return;+ Interner<String> interner = Interner.newWeakInterner();+ ClusterState.setStrInternerParser(+ new Function<>() {+ @Override+ public ObjectBuilder apply(JSONParser p) {+ try {+ return new ObjectBuilder(p) {+ @Override+ public void addKeyVal(Object map, Object key, Object val) throws IOException {+ if (key != null) {+ key = interner.intern(key.toString());+ }+ if (val instanceof String) {+ val = interner.intern((String) val);+ }+ super.addKeyVal(map, key, val);+ }+ };+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ }+ });+ }}class CloserThread extends Thread {diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.javaindex d54446afa0d..7d7e3c30a12 100644--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java@@ -16,6 +16,8 @@*/package org.apache.solr.common.cloud;+import static org.apache.solr.common.util.Utils.STANDARDOBJBUILDER;+import java.lang.invoke.MethodHandles;import java.util.Collection;import java.util.Collections;@@ -28,6 +30,7 @@ import java.util.Optional;import java.util.Set;import java.util.concurrent.atomic.AtomicInteger;import java.util.function.Consumer;+import java.util.function.Function;import java.util.function.Supplier;import java.util.stream.Collectors;import org.apache.solr.common.SolrException;@@ -35,7 +38,9 @@ import org.apache.solr.common.SolrException.ErrorCode;import org.apache.solr.common.cloud.DocCollection.CollectionStateProps;import org.apache.solr.common.cloud.Replica.ReplicaStateProps;import org.apache.solr.common.util.Utils;+import org.noggit.JSONParser;import org.noggit.JSONWriter;+import org.noggit.ObjectBuilder;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -227,7 +232,8 @@ public class ClusterState implements JSONWriter.Writable {return new ClusterState(liveNodes, Collections.<String, DocCollection>emptyMap());}@SuppressWarnings({"unchecked"})- Map<String, Object> stateMap = (Map<String, Object>) Utils.fromJSON(bytes);+ Map<String, Object> stateMap =+ (Map<String, Object>) Utils.fromJSON(bytes, 0, bytes.length, STR_INTERNER_OBJ_BUILDER);return createFromCollectionMap(version, stateMap, liveNodes);}@@ -477,4 +483,12 @@ public class ClusterState implements JSONWriter.Writable {return perReplicaStates;}}++ private static volatile Function<JSONParser, ObjectBuilder> STR_INTERNER_OBJ_BUILDER =+ STANDARDOBJBUILDER;++ public static void setStrInternerParser(Function<JSONParser, ObjectBuilder> fun) {+ if (fun == null) return;+ STR_INTERNER_OBJ_BUILDER = fun;+ }}