Apache Solr open-source search software
SOLR-17006: Collections/AddReplica: Persist user-defined properties to state.json (#1973)
Vincent P0f5a8f929ee4
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex dc634381da6..14330008e72 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -73,6 +73,9 @@ New Features---------------------* SOLR-14496: Solr CLI commands now can interact with a Solr secured using Basic Authentication. (Eric Pugh)+* SOLR-17006: Collection creation & adding replicas: User-defined properties are persisted to state.json and+ applied to new replicas, available for use as property substitution in configuration files. (Vincent Primault)+Improvements---------------------* SOLR-16924: RESTORECORE now sets the UpdateLog to ACTIVE state instead of requiring a separatediff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.javaindex 08dc391293f..e7c66689874 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java@@ -39,6 +39,7 @@ import java.util.Collection;import java.util.Collections;import java.util.List;import java.util.Locale;+import java.util.Map;import java.util.concurrent.TimeUnit;import java.util.stream.Collectors;import org.apache.solr.client.solrj.cloud.SolrCloudManager;@@ -252,10 +253,10 @@ public class AddReplicaCmd implements CollApiCmds.CollectionApiCommand {CreateReplica createReplica)throws InterruptedException, KeeperException {if (!skipCreateReplicaInClusterState) {- ZkNodeProps props =- new ZkNodeProps(+ Map<String, Object> replicaProps =+ Utils.makeMap(Overseer.QUEUE_OPERATION,- ADDREPLICA.toLower(),+ (Object) ADDREPLICA.toLower(),ZkStateReader.COLLECTION_PROP,collectionName,ZkStateReader.SHARD_ID_PROP,@@ -271,8 +272,11 @@ public class AddReplicaCmd implements CollApiCmds.CollectionApiCommand {ZkStateReader.REPLICA_TYPE,createReplica.replicaType.name());if (createReplica.coreNodeName != null) {- props = props.plus(ZkStateReader.CORE_NODE_NAME_PROP, createReplica.coreNodeName);+ replicaProps.put(ZkStateReader.CORE_NODE_NAME_PROP, createReplica.coreNodeName);}+ CollectionHandlingUtils.addPropertyParams(message, replicaProps);++ ZkNodeProps props = new ZkNodeProps(replicaProps);if (ccc.getDistributedClusterStateUpdater().isDistributedStateUpdate()) {ccc.getDistributedClusterStateUpdater().doSingleStateUpdate(@@ -339,6 +343,9 @@ public class AddReplicaCmd implements CollApiCmds.CollectionApiCommand {if (createReplica.coreNodeName != null) {params.set(CoreAdminParams.CORE_NODE_NAME, createReplica.coreNodeName);}+ // Inherit user-defined properties from collection.+ CollectionHandlingUtils.addPropertyParams(coll, params);+ // Inherit user-defined properties from replica.CollectionHandlingUtils.addPropertyParams(message, params);return params;diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ClusterStateMutator.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ClusterStateMutator.javaindex 2c625b2c5c4..11b3795745b 100644--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ClusterStateMutator.java+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ClusterStateMutator.java@@ -125,6 +125,13 @@ public class ClusterStateMutator {collectionProps.put(ZkStateReader.CONFIGNAME_PROP, configName);}+ // add user-defined properties+ for (String prop : message.keySet()) {+ if (prop.startsWith(CollectionAdminParams.PROPERTY_PREFIX)) {+ collectionProps.put(prop, message.get(prop));+ }+ }+assert !collectionProps.containsKey(CollectionAdminParams.COLL_CONF);DocCollection newCollection =DocCollection.create(diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/SliceMutator.java b/solr/core/src/java/org/apache/solr/cloud/overseer/SliceMutator.javaindex c83a965d5a1..c016fa5489b 100644--- a/solr/core/src/java/org/apache/solr/cloud/overseer/SliceMutator.java+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/SliceMutator.java@@ -94,25 +94,30 @@ public class SliceMutator {cloudManager.getClusterStateProvider().getClusterProperty(ZkStateReader.URL_SCHEME, "http"));- Replica replica =- new Replica(- coreNodeName,- Utils.makeMap(- ZkStateReader.CORE_NAME_PROP,- message.getStr(ZkStateReader.CORE_NAME_PROP),- ZkStateReader.STATE_PROP,- message.getStr(ZkStateReader.STATE_PROP),- ZkStateReader.NODE_NAME_PROP,- nodeName,- ZkStateReader.BASE_URL_PROP,- baseUrl,- ZkStateReader.FORCE_SET_STATE_PROP,- "false",- ZkStateReader.REPLICA_TYPE,- message.get(ZkStateReader.REPLICA_TYPE)),- coll,- slice);+ Map<String, Object> replicaProps =+ Utils.makeMap(+ ZkStateReader.CORE_NAME_PROP,+ message.getStr(ZkStateReader.CORE_NAME_PROP),+ ZkStateReader.STATE_PROP,+ message.getStr(ZkStateReader.STATE_PROP),+ ZkStateReader.NODE_NAME_PROP,+ nodeName,+ ZkStateReader.BASE_URL_PROP,+ baseUrl,+ ZkStateReader.FORCE_SET_STATE_PROP,+ "false",+ ZkStateReader.REPLICA_TYPE,+ message.get(ZkStateReader.REPLICA_TYPE));++ // add user-defined properties+ for (String prop : message.keySet()) {+ if (prop.startsWith(CollectionAdminParams.PROPERTY_PREFIX)) {+ replicaProps.put(prop, message.get(prop));+ }+ }++ Replica replica = new Replica(coreNodeName, replicaProps, coll, slice);return new ZkWriteCommand(coll, updateReplica(collection, sl, replica.getName(), replica));}diff --git a/solr/core/src/test/org/apache/solr/cloud/AddReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/AddReplicaTest.javaindex 7702012713d..fa2bdf84695 100644--- a/solr/core/src/test/org/apache/solr/cloud/AddReplicaTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/AddReplicaTest.java@@ -27,6 +27,7 @@ import org.apache.solr.client.solrj.response.RequestStatusState;import org.apache.solr.common.cloud.ClusterState;import org.apache.solr.common.cloud.DocCollection;import org.apache.solr.common.cloud.Replica;+import org.apache.solr.core.CoreDescriptor;import org.junit.Before;import org.junit.BeforeClass;import org.junit.Test;@@ -187,4 +188,37 @@ public class AddReplicaTest extends SolrCloudTestCase {assertSame(coll + "\n" + replica, replica.getState(), Replica.State.ACTIVE);}}++ @Test+ public void testAddReplicaWithUserDefinedProperties() throws Exception {+ // When creating a collection with user-defined properties+ CloudSolrClient cloudClient = cluster.getSolrClient();+ String collectionName = "testAddReplicaWithUserDefinedProperties";+ CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1)+ .withProperty("customProp1", "val1")+ .withProperty("customProp2", "val2")+ .process(cloudClient);+ cluster.waitForActiveCollection(collectionName, 1, 1);++ // When adding a replica to the collection with user-defined properties+ CollectionAdminRequest.AddReplica addReplica =+ CollectionAdminRequest.addReplicaToShard(collectionName, "shard1");+ addReplica.withProperty("customProp2", "val2.1");+ addReplica.withProperty("customProp3", "val3");+ addReplica.setWaitForFinalState(true);+ addReplica.process(cloudClient);++ // Verify that the new core was created with user-defined properties coming from the request+ // and inherited from the collection (the former taking precedence over the latter).+ Replica replica =+ cloudClient.getClusterState().getCollection(collectionName).getReplicas().get(1);+ CoreDescriptor coreDescriptor =+ cluster+ .getReplicaJetty(replica)+ .getCoreContainer()+ .getCoreDescriptor(replica.getCoreName());+ assertEquals("val1", coreDescriptor.getCoreProperty("customProp1", ""));+ assertEquals("val2.1", coreDescriptor.getCoreProperty("customProp2", ""));+ assertEquals("val3", coreDescriptor.getCoreProperty("customProp3", ""));+ }}diff --git a/solr/core/src/test/org/apache/solr/cloud/CreateCollectionWithPropertiesTest.java b/solr/core/src/test/org/apache/solr/cloud/CreateCollectionWithPropertiesTest.javanew file mode 100644index 00000000000..c90e9d21190--- /dev/null+++ b/solr/core/src/test/org/apache/solr/cloud/CreateCollectionWithPropertiesTest.java@@ -0,0 +1,68 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.cloud;++import org.apache.solr.client.solrj.impl.CloudSolrClient;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.common.cloud.DocCollection;+import org.apache.solr.common.cloud.Replica;+import org.apache.solr.core.CoreDescriptor;+import org.junit.Before;+import org.junit.BeforeClass;+import org.junit.Test;++public class CreateCollectionWithPropertiesTest extends SolrCloudTestCase {+ @BeforeClass+ public static void setupCluster() throws Exception {+ configureCluster(3)+ .addConfig(+ "conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))+ .configure();+ }++ @Override+ @Before+ public void setUp() throws Exception {+ super.setUp();+ cluster.deleteAllCollections();+ }++ @Test+ public void testCreateCollectionWithUserDefinedProperties() throws Exception {+ // When creating a collection with user-defined properties+ CloudSolrClient cloudClient = cluster.getSolrClient();+ String collectionName = "testCreateCollectionWithUserDefinedProperties";+ CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1)+ .withProperty("customProp1", "val1")+ .process(cloudClient);+ cluster.waitForActiveCollection(collectionName, 1, 1);++ // Verify that user-defined properties are stored in cluster state+ DocCollection collection = cloudClient.getClusterState().getCollection(collectionName);+ assertEquals("val1", collection.getProperties().get("property.customProp1"));++ // Verify that the core was created with user-defined properties+ Replica replica =+ cloudClient.getClusterState().getCollection(collectionName).getReplicas().get(0);+ CoreDescriptor coreDescriptor =+ cluster+ .getReplicaJetty(replica)+ .getCoreContainer()+ .getCoreDescriptor(replica.getCoreName());+ assertEquals("val1", coreDescriptor.getCoreProperty("customProp1", ""));+ }+}diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adocindex 107b4849209..63da9c86e80 100644--- a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc@@ -250,6 +250,7 @@ If `true` the states of individual replicas will be maintained as individual chi+Set core property _name_ to _value_.See the section xref:configuration-guide:core-discovery.adoc[] for details on supported properties and values.+Those properties are also applied to every new core that will be created when adding replicas to the collection later on.+[WARNING]====