Apache Solr open-source search software
SOLR-11904: Mark ReplicationHandler's polling thread as a Solr server thread so the PKI Interceptor is activated to allow PULL replicas to replicate from security-enabled leaders (#112)
Timothy Potterae1ac22eb70d
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex e2ebaecb7bb..ea1031041ff 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -381,6 +381,9 @@ Bug Fixes* SOLR-15383: Solr Zookeeper status page shows green even when some Zookeepers are not serving requests (janhoy)+* SOLR-11904: Mark ReplicationHandler's polling thread as a Solr server thread so the PKI Interceptor is activated to+ allow PULL replicas to replicate from security-enabled leaders (Timothy Potter, Torsten Bøgh Köster)+Other Changes---------------------* SOLR-15118: Deprecate CollectionAdminRequest.getV2Request(). (Jason Gerlowski)diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.javaindex 414d3c75897..4572be122ed 100644--- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java@@ -1208,6 +1208,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAwlog.info("Poll disabled");return;}+ ExecutorUtil.setServerThreadFlag(true); // so PKI auth workstry {log.debug("Polling for index modifications");markScheduledExecutionStart();@@ -1215,6 +1216,8 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAwif (pollListener != null) pollListener.onComplete(core, fetchResult);} catch (Exception e) {log.error("Exception in fetching index", e);+ } finally {+ ExecutorUtil.setServerThreadFlag(null);}};executorService = Executors.newSingleThreadScheduledExecutor(diff --git a/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java b/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.javaindex 5e06c28b5b9..1bbe258a22f 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java@@ -28,19 +28,19 @@ import java.util.Map;import java.util.concurrent.TimeUnit;import java.util.stream.Collectors;+import com.carrotsearch.randomizedtesting.annotations.Repeat;import org.apache.http.HttpResponse;-import org.apache.http.client.ClientProtocolException;import org.apache.http.client.HttpClient;import org.apache.http.client.methods.HttpGet;import org.apache.http.client.methods.HttpPost;import org.apache.http.entity.StringEntity;-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;import org.apache.lucene.util.LuceneTestCase.Slow;import org.apache.solr.client.solrj.SolrQuery;import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.embedded.JettySolrRunner;import org.apache.solr.client.solrj.impl.HttpSolrClient;import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.request.QueryRequest;import org.apache.solr.client.solrj.response.CollectionAdminResponse;import org.apache.solr.client.solrj.response.QueryResponse;import org.apache.solr.common.SolrDocument;@@ -54,6 +54,7 @@ import org.apache.solr.common.cloud.Slice;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.util.TimeSource;import org.apache.solr.core.SolrCore;+import org.apache.solr.util.LogLevel;import org.apache.solr.util.TestInjection;import org.apache.solr.util.TimeOut;import org.apache.zookeeper.KeeperException;@@ -64,10 +65,8 @@ import org.junit.Test;import org.slf4j.Logger;import org.slf4j.LoggerFactory;-import com.carrotsearch.randomizedtesting.annotations.Repeat;-@Slow-@AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")+@LogLevel("org.apache.solr.handler.ReplicationHandler=DEBUG,org.apache.solr.handler.IndexFetcher=DEBUG")public class TestPullReplica extends SolrCloudTestCase {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());@@ -80,7 +79,7 @@ public class TestPullReplica extends SolrCloudTestCase {}@BeforeClass- public static void setupCluster() throws Exception {+ public static void createTestCluster() throws Exception {// cloudSolrClientMaxStaleRetriesSystem.setProperty("cloudSolrClientMaxStaleRetries", "1");System.setProperty("zkReaderGetLeaderRetryTimeoutMs", "1000");@@ -170,7 +169,7 @@ public class TestPullReplica extends SolrCloudTestCase {2, docCollection.getReplicas(EnumSet.of(Replica.Type.NRT)).size());for (Slice s:docCollection.getSlices()) {// read-only replicas can never become leaders- assertFalse(s.getLeader().getType() == Replica.Type.PULL);+ assertNotSame(s.getLeader().getType(), Replica.Type.PULL);List<String> shardElectionNodes = cluster.getZkClient().getChildren(ZkStateReader.getShardLeadersElectPath(collectionName, s.getName()), null, true);assertEquals("Unexpected election nodes for Shard: " + s.getName() + ": " + Arrays.toString(shardElectionNodes.toArray()),1, shardElectionNodes.size());@@ -194,27 +193,22 @@ public class TestPullReplica extends SolrCloudTestCase {/*** Asserts that Update logs don't exist for replicas of type {@link org.apache.solr.common.cloud.Replica.Type#PULL}*/- private void assertUlogPresence(DocCollection collection) {+ static void assertUlogPresence(DocCollection collection) {for (Slice s:collection.getSlices()) {for (Replica r:s.getReplicas()) {if (r.getType() == Replica.Type.NRT) {continue;}- SolrCore core = null;- try {- core = cluster.getReplicaJetty(r).getCoreContainer().getCore(r.getCoreName());+ try (SolrCore core = cluster.getReplicaJetty(r).getCoreContainer().getCore(r.getCoreName())) {assertNotNull(core);assertFalse("Update log should not exist for replicas of type Passive but file is present: " + core.getUlogDir(),new java.io.File(core.getUlogDir()).exists());- } finally {- core.close();}}}}@SuppressWarnings("unchecked")- // 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")public void testAddDocs() throws Exception {int numPullReplicas = 1 + random().nextInt(3);CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1, 0, numPullReplicas)@@ -229,37 +223,27 @@ public class TestPullReplica extends SolrCloudTestCase {numDocs++;cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", String.valueOf(numDocs), "foo", "bar"));cluster.getSolrClient().commit(collectionName);+ log.info("Committed doc {} to leader", numDocs);Slice s = docCollection.getSlices().iterator().next();try (HttpSolrClient leaderClient = getHttpSolrClient(s.getLeader().getCoreUrl())) {assertEquals(numDocs, leaderClient.query(new SolrQuery("*:*")).getResults().getNumFound());}+ log.info("Found {} docs in leader, verifying updates make it to {} pull replicas", numDocs, numPullReplicas);- TimeOut t = new TimeOut(REPLICATION_TIMEOUT_SECS, TimeUnit.SECONDS, TimeSource.NANO_TIME);- for (Replica r:s.getReplicas(EnumSet.of(Replica.Type.PULL))) {- //TODO: assert replication < REPLICATION_TIMEOUT_SECS+ List<Replica> pullReplicas = s.getReplicas(EnumSet.of(Replica.Type.PULL));+ waitForNumDocsInAllReplicas(numDocs, pullReplicas);++ for (Replica r : pullReplicas) {try (HttpSolrClient pullReplicaClient = getHttpSolrClient(r.getCoreUrl())) {- while (true) {- try {- assertEquals("Replica " + r.getName() + " not up to date after 10 seconds",- numDocs, pullReplicaClient.query(new SolrQuery("*:*")).getResults().getNumFound());- break;- } catch (AssertionError e) {- if (t.hasTimedOut()) {- throw e;- } else {- Thread.sleep(100);- }- }- }- SolrQuery req = new SolrQuery(- "qt", "/admin/plugins",- "stats", "true");+ SolrQuery req = new SolrQuery("qt", "/admin/plugins", "stats", "true");QueryResponse statsResponse = pullReplicaClient.query(req);- assertEquals("Replicas shouldn't process the add document request: " + statsResponse,- 0L, ((Map<String, Object>)(statsResponse.getResponse()).findRecursive("plugins", "UPDATE", "updateHandler", "stats")).get("UPDATE.updateHandler.adds"));+ // The adds gauge metric should be null for pull replicas since they don't process adds+ assertNull("Replicas shouldn't process the add document request: " + statsResponse,+ ((Map<String, Object>)(statsResponse.getResponse()).findRecursive("plugins", "UPDATE", "updateHandler", "stats")).get("UPDATE.updateHandler.adds"));}}+if (reloaded) {break;} else {@@ -445,7 +429,7 @@ public class TestPullReplica extends SolrCloudTestCase {// Also fails if I send the update to the pull replica explicitlytry (HttpSolrClient pullReplicaClient = getHttpSolrClient(docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)).get(0).getCoreUrl())) {expectThrows(SolrException.class, () ->- cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "2", "foo", "zoo"))+ pullReplicaClient.add(collectionName, new SolrInputDocument("id", "2", "foo", "zoo")));}if (removeReplica) {@@ -486,7 +470,7 @@ public class TestPullReplica extends SolrCloudTestCase {leaderClient.commit();assertEquals(1, leaderClient.query(new SolrQuery("*:*")).getResults().getNumFound());}- waitForNumDocsInAllReplicas(1, docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)), "id:2");+ waitForNumDocsInAllReplicas(1, docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)), "id:2", null, null);waitForNumDocsInAllReplicas(1, docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)));}@@ -528,23 +512,30 @@ public class TestPullReplica extends SolrCloudTestCase {}private void waitForNumDocsInAllReplicas(int numDocs, Collection<Replica> replicas) throws IOException, SolrServerException, InterruptedException {- waitForNumDocsInAllReplicas(numDocs, replicas, "*:*");+ waitForNumDocsInAllReplicas(numDocs, replicas, "*:*", null, null);}- private void waitForNumDocsInAllReplicas(int numDocs, Collection<Replica> replicas, String query) throws IOException, SolrServerException, InterruptedException {+ static void waitForNumDocsInAllReplicas(int numDocs, Collection<Replica> replicas, String query, String user, String pass) throws IOException, SolrServerException, InterruptedException {TimeOut t = new TimeOut(REPLICATION_TIMEOUT_SECS, TimeUnit.SECONDS, TimeSource.NANO_TIME);for (Replica r:replicas) {- try (HttpSolrClient replicaClient = getHttpSolrClient(r.getCoreUrl())) {+ String replicaUrl = r.getCoreUrl();+ try (HttpSolrClient replicaClient = getHttpSolrClient(replicaUrl)) {while (true) {+ QueryRequest req = new QueryRequest(new SolrQuery(query));+ if (user != null && pass != null) {+ req.setBasicAuthCredentials(user, pass);+ }try {- assertEquals("Replica " + r.getName() + " not up to date after " + REPLICATION_TIMEOUT_SECS + " seconds",- numDocs, replicaClient.query(new SolrQuery(query)).getResults().getNumFound());+ long numFound = req.process(replicaClient).getResults().getNumFound();+ assertEquals("Replica " + r.getName() + " (" + replicaUrl + ") not up to date after " + REPLICATION_TIMEOUT_SECS + " seconds",+ numDocs, numFound);+ log.info("Replica {} ({}) has all {} docs", r.name, replicaUrl, numDocs);break;} catch (AssertionError e) {if (t.hasTimedOut()) {throw e;} else {- Thread.sleep(100);+ Thread.sleep(200);}}}@@ -552,7 +543,7 @@ public class TestPullReplica extends SolrCloudTestCase {}}- private void waitForDeletion(String collection) throws InterruptedException, KeeperException {+ static void waitForDeletion(String collection) throws InterruptedException, KeeperException {TimeOut t = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);while (cluster.getSolrClient().getZkStateReader().getClusterState().hasCollection(collection)) {log.info("Collection not yet deleted");@@ -570,10 +561,14 @@ public class TestPullReplica extends SolrCloudTestCase {}private DocCollection assertNumberOfReplicas(int numNrtReplicas, int numTlogReplicas, int numPullReplicas, boolean updateCollection, boolean activeOnly) throws KeeperException, InterruptedException {+ return assertNumberOfReplicas(collectionName, numNrtReplicas, numTlogReplicas, numPullReplicas, updateCollection, activeOnly);+ }++ static DocCollection assertNumberOfReplicas(String coll, int numNrtReplicas, int numTlogReplicas, int numPullReplicas, boolean updateCollection, boolean activeOnly) throws KeeperException, InterruptedException {if (updateCollection) {- cluster.getSolrClient().getZkStateReader().forceUpdateCollection(collectionName);+ cluster.getSolrClient().getZkStateReader().forceUpdateCollection(coll);}- DocCollection docCollection = getCollectionState(collectionName);+ DocCollection docCollection = getCollectionState(coll);assertNotNull(docCollection);assertEquals("Unexpected number of writer replicas: " + docCollection, numNrtReplicas,docCollection.getReplicas(EnumSet.of(Replica.Type.NRT)).stream().filter(r->!activeOnly || r.getState() == Replica.State.ACTIVE).count());@@ -641,7 +636,7 @@ public class TestPullReplica extends SolrCloudTestCase {cluster.getSolrClient().commit(collectionName);}- private void addReplicaToShard(String shardName, Replica.Type type) throws ClientProtocolException, IOException, SolrServerException {+ private void addReplicaToShard(String shardName, Replica.Type type) throws IOException, SolrServerException {switch (random().nextInt(3)) {case 0: // Add replica with SolrJCollectionAdminResponse response = CollectionAdminRequest.addReplicaToShard(collectionName, shardName, type).process(cluster.getSolrClient());diff --git a/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaWithAuth.java b/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaWithAuth.javanew file mode 100644index 00000000000..b5cb3700299--- /dev/null+++ b/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaWithAuth.java@@ -0,0 +1,156 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.cloud;++import java.io.IOException;+import java.util.EnumSet;+import java.util.List;+import java.util.Map;++import org.apache.lucene.util.LuceneTestCase.Slow;+import org.apache.solr.client.solrj.SolrQuery;+import org.apache.solr.client.solrj.SolrRequest;+import org.apache.solr.client.solrj.SolrResponse;+import org.apache.solr.client.solrj.SolrServerException;+import org.apache.solr.client.solrj.impl.CloudSolrClient;+import org.apache.solr.client.solrj.impl.HttpSolrClient;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.request.QueryRequest;+import org.apache.solr.client.solrj.request.UpdateRequest;+import org.apache.solr.client.solrj.response.CollectionAdminResponse;+import org.apache.solr.client.solrj.response.QueryResponse;+import org.apache.solr.common.SolrInputDocument;+import org.apache.solr.common.cloud.DocCollection;+import org.apache.solr.common.cloud.Replica;+import org.apache.solr.common.cloud.Slice;+import org.apache.solr.common.util.Utils;+import org.apache.solr.security.BasicAuthPlugin;+import org.apache.solr.security.RuleBasedAuthorizationPlugin;+import org.junit.BeforeClass;++import static java.util.Collections.singletonList;+import static java.util.Collections.singletonMap;+import static org.apache.solr.cloud.TestPullReplica.assertNumberOfReplicas;+import static org.apache.solr.cloud.TestPullReplica.assertUlogPresence;+import static org.apache.solr.cloud.TestPullReplica.waitForDeletion;+import static org.apache.solr.cloud.TestPullReplica.waitForNumDocsInAllReplicas;+import static org.apache.solr.security.Sha256AuthenticationProvider.getSaltedHashedValue;++@Slow+public class TestPullReplicaWithAuth extends SolrCloudTestCase {++ private static final String USER = "solr";+ private static final String PASS = "SolrRocksAgain";+ private static final String collectionName = "testPullReplicaWithAuth";++ @BeforeClass+ public static void setupClusterWithSecurityEnabled() throws Exception {+ final String SECURITY_JSON = Utils.toJSONString+ (Map.of("authorization",+ Map.of("class", RuleBasedAuthorizationPlugin.class.getName(),+ "user-role", singletonMap(USER, "admin"),+ "permissions", singletonList(Map.of("name", "all", "role", "admin"))),+ "authentication",+ Map.of("class", BasicAuthPlugin.class.getName(),+ "blockUnknown", true,+ "credentials", singletonMap(USER, getSaltedHashedValue(PASS)))));++ configureCluster(2)+ .addConfig("conf", configset("cloud-minimal"))+ .withSecurityJson(SECURITY_JSON)+ .configure();+ }++ private <T extends SolrRequest<? extends SolrResponse>> T withBasicAuth(T req) {+ req.setBasicAuthCredentials(USER, PASS);+ return req;+ }++ private QueryResponse queryWithBasicAuth(HttpSolrClient client, SolrQuery q) throws IOException, SolrServerException {+ return withBasicAuth(new QueryRequest(q)).process(client);+ }++ public void testPKIAuthWorksForPullReplication() throws Exception {+ int numPullReplicas = 2;+ withBasicAuth(CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1, 0, numPullReplicas))+ .process(cluster.getSolrClient());+ waitForState("Expected collection to be created with 1 shard and " + (numPullReplicas + 1) + " replicas",+ collectionName, clusterShape(1, numPullReplicas + 1));+ DocCollection docCollection =+ assertNumberOfReplicas(collectionName, 1, 0, numPullReplicas, false, true);++ int numDocs = 0;+ CloudSolrClient solrClient = cluster.getSolrClient();+ for (int i = 0; i < 5; i++) {+ numDocs++;++ UpdateRequest ureq = withBasicAuth(new UpdateRequest());+ ureq.add(new SolrInputDocument("id", String.valueOf(numDocs), "foo", "bar"));+ ureq.commit(solrClient, collectionName);++ Slice s = docCollection.getSlices().iterator().next();+ try (HttpSolrClient leaderClient = getHttpSolrClient(s.getLeader().getCoreUrl())) {+ assertEquals(numDocs, queryWithBasicAuth(leaderClient, new SolrQuery("*:*")).getResults().getNumFound());+ }++ List<Replica> pullReplicas = s.getReplicas(EnumSet.of(Replica.Type.PULL));+ waitForNumDocsInAllReplicas(numDocs, pullReplicas, "*:*", USER, PASS);++ for (Replica r : pullReplicas) {+ try (HttpSolrClient pullReplicaClient = getHttpSolrClient(r.getCoreUrl())) {+ QueryResponse statsResponse = queryWithBasicAuth(pullReplicaClient, new SolrQuery("qt", "/admin/plugins", "stats", "true"));+ // adds is a gauge, which is null for PULL replicas+ assertNull("Replicas shouldn't process the add document request: " + statsResponse,+ getUpdateHandlerMetric(statsResponse, "UPDATE.updateHandler.adds"));+ assertEquals("Replicas shouldn't process the add document request: " + statsResponse,+ 0L, getUpdateHandlerMetric(statsResponse, "UPDATE.updateHandler.cumulativeAdds.count"));+ }+ }+ }++ CollectionAdminResponse response =+ withBasicAuth(CollectionAdminRequest.reloadCollection(collectionName)).process(cluster.getSolrClient());+ assertEquals(0, response.getStatus());+ assertUlogPresence(docCollection);++ // add another pull replica to ensure it can pull the indexes+ Slice s = docCollection.getSlices().iterator().next();+ List<Replica> pullReplicas = s.getReplicas(EnumSet.of(Replica.Type.PULL));+ assertEquals(numPullReplicas, pullReplicas.size());+ response = withBasicAuth(CollectionAdminRequest.addReplicaToShard(collectionName, s.getName(), Replica.Type.PULL)).process(cluster.getSolrClient());+ assertEquals(0, response.getStatus());++ numPullReplicas = numPullReplicas + 1; // added a PULL+ waitForState("Expected collection to be created with 1 shard and " + (numPullReplicas + 1) + " replicas",+ collectionName, clusterShape(1, numPullReplicas + 1));++ docCollection =+ assertNumberOfReplicas(collectionName, 1, 0, numPullReplicas, false, true);+ s = docCollection.getSlices().iterator().next();+ pullReplicas = s.getReplicas(EnumSet.of(Replica.Type.PULL));+ assertEquals(numPullReplicas, pullReplicas.size());+ waitForNumDocsInAllReplicas(numDocs, pullReplicas, "*:*", USER, PASS);++ withBasicAuth(CollectionAdminRequest.deleteCollection(collectionName)).process(cluster.getSolrClient());+ waitForDeletion(collectionName);+ }++ @SuppressWarnings("unchecked")+ private Object getUpdateHandlerMetric(QueryResponse statsResponse, String metric) {+ return ((Map<String, Object>) statsResponse.getResponse().findRecursive("plugins", "UPDATE", "updateHandler", "stats")).get(metric);+ }+}