Apache Solr open-source search software
SOLR-17568: SolrCloud shouldn't proxy/route a core specific request (#2885)
David Smiley3e358911be52
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex a7a5ef5683c..9a5f4aae64b 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -36,7 +36,8 @@ ImprovementsOptimizations----------------------(No changes)+* SOLR-17568: The CLI bin/solr export tool now contacts the appropriate nodes directly for data instead of proxying through one.+ (David Smiley)Bug Fixes---------------------@@ -132,6 +133,8 @@ Other Changes* SOLR-16903: Update CLI tools to use java.nio.file.Path instead of java.io.File (Andrey Bozhko)+* SOLR-17568: SolrCloud no longer reroutes/proxies a core request to another node if not found locally. (David Smiley)+================== 9.8.0 ==================New Features---------------------diff --git a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java b/solr/core/src/java/org/apache/solr/api/V2HttpCall.javaindex 6912af39510..d156710a675 100644--- a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java+++ b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java@@ -165,7 +165,7 @@ public class V2HttpCall extends HttpSolrCall {core = getCoreByCollection(collectionName, isPreferLeader);if (core == null) {// this collection exists , but this node does not have a replica for that collection- extractRemotePath(collectionName, collectionName);+ extractRemotePath(collectionName);if (action == REMOTEQUERY) {action = ADMIN_OR_REMOTEQUERY;coreUrl = coreUrl.replace("/solr/", "/solr/____v2/c/");diff --git a/solr/core/src/java/org/apache/solr/cli/ExportTool.java b/solr/core/src/java/org/apache/solr/cli/ExportTool.javaindex 59bca949dc1..b5a58377ba3 100644--- a/solr/core/src/java/org/apache/solr/cli/ExportTool.java+++ b/solr/core/src/java/org/apache/solr/cli/ExportTool.java@@ -660,8 +660,8 @@ public class ExportTool extends ToolBase {}boolean exportDocsFromCore() throws IOException, SolrServerException {-- try (SolrClient client = CLIUtils.getSolrClient(baseurl, credentials)) {+ // reference the replica's node URL, not the baseUrl in scope, which could be anywhere+ try (SolrClient client = CLIUtils.getSolrClient(replica.getBaseUrl(), credentials)) {expectedDocs = getDocCount(replica.getCoreName(), client, query);QueryRequest request;ModifiableSolrParams params = new ModifiableSolrParams();diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.javaindex 7241be77390..1e6c9f42e29 100644--- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java+++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java@@ -40,7 +40,6 @@ import java.io.OutputStream;import java.lang.invoke.MethodHandles;import java.nio.charset.StandardCharsets;import java.util.ArrayList;-import java.util.Arrays;import java.util.Collection;import java.util.Collections;import java.util.Enumeration;@@ -279,7 +278,7 @@ public class HttpSolrCall {} else {// if we couldn't find it locally, look on other nodesif (idx > 0) {- extractRemotePath(collectionName, origCorename);+ extractRemotePath(collectionName);if (action == REMOTEQUERY) {path = path.substring(idx);return;@@ -462,10 +461,10 @@ public class HttpSolrCall {}}- protected void extractRemotePath(String collectionName, String origCorename)+ protected void extractRemotePath(String collectionName)throws KeeperException, InterruptedException, SolrException {assert core == null;- coreUrl = getRemoteCoreUrl(collectionName, origCorename);+ coreUrl = getRemoteCoreUrl(collectionName);// don't proxy for internal update requestsinvalidStates = checkStateVersionsAreValid(queryParams.get(CloudSolrClient.STATE_VERSION));if (coreUrl != null@@ -1090,39 +1089,16 @@ public class HttpSolrCall {return core;}- private List<Slice> getSlicesForAllCollections(ClusterState clusterState, boolean activeSlices) {- // looks across *all* collections- if (activeSlices) {- return clusterState- .collectionStream()- .flatMap(coll -> Arrays.stream(coll.getActiveSlicesArr()))- .toList();- } else {- return clusterState.collectionStream().flatMap(coll -> coll.getSlices().stream()).toList();- }- }-- protected String getRemoteCoreUrl(String collectionName, String origCorename)- throws SolrException {+ protected String getRemoteCoreUrl(String collectionName) throws SolrException {ClusterState clusterState = cores.getZkController().getClusterState();final DocCollection docCollection = clusterState.getCollectionOrNull(collectionName);- Slice[] slices = (docCollection != null) ? docCollection.getActiveSlicesArr() : null;- List<Slice> activeSlices;- boolean byCoreName = false;+ if (docCollection == null) {+ return null;+ }+ Collection<Slice> activeSlices = docCollection.getActiveSlices();int totalReplicas = 0;- if (slices == null) {- byCoreName = true;- // all collections!- activeSlices = getSlicesForAllCollections(clusterState, true);- if (activeSlices.isEmpty()) {- activeSlices = getSlicesForAllCollections(clusterState, false);- }- } else {- activeSlices = List.of(slices);- }-for (Slice s : activeSlices) {totalReplicas += s.getReplicas().size();}@@ -1145,48 +1121,30 @@ public class HttpSolrCall {"No active replicas found for collection: " + collectionName);}- String coreUrl =- getCoreUrl(collectionName, origCorename, clusterState, activeSlices, byCoreName, true);+ String coreUrl = getCoreUrl(activeSlices, true, clusterState.getLiveNodes());if (coreUrl == null) {- coreUrl =- getCoreUrl(collectionName, origCorename, clusterState, activeSlices, byCoreName, false);+ coreUrl = getCoreUrl(activeSlices, false, clusterState.getLiveNodes());}return coreUrl;}private String getCoreUrl(- String collectionName,- String origCorename,- ClusterState clusterState,- List<Slice> slices,- boolean byCoreName,- boolean activeReplicas) {- String coreUrl;- Set<String> liveNodes = clusterState.getLiveNodes();+ Collection<Slice> slices, boolean activeReplicas, Set<String> liveNodes) {- List<Slice> shuffledSlices;- if (slices.size() < 2) {- shuffledSlices = slices;- } else {- shuffledSlices = new ArrayList<>(slices);- Collections.shuffle(shuffledSlices, Utils.RANDOM);- }+ Iterator<Slice> shuffledSlices = new RandomIterator<>(Utils.RANDOM, slices);+ while (shuffledSlices.hasNext()) {+ Slice slice = shuffledSlices.next();- for (Slice slice : shuffledSlices) {- List<Replica> randomizedReplicas = new ArrayList<>(slice.getReplicas());- Collections.shuffle(randomizedReplicas, Utils.RANDOM);+ Iterator<Replica> shuffledReplicas = new RandomIterator<>(Utils.RANDOM, slice.getReplicas());+ while (shuffledReplicas.hasNext()) {+ Replica replica = shuffledReplicas.next();- for (Replica replica : randomizedReplicas) {if (!activeReplicas|| (liveNodes.contains(replica.getNodeName())&& replica.getState() == Replica.State.ACTIVE)) {- if (byCoreName && !Objects.equals(origCorename, replica.getStr(CORE_NAME_PROP))) {- // if it's by core name, make sure they match- continue;- }if (Objects.equals(replica.getBaseUrl(), cores.getZkController().getBaseUrl())) {// don't count a local corecontinue;diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adocindex 97942e989c1..2d1902a02fa 100644--- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc+++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc@@ -60,6 +60,12 @@ Users who previously relied on collection-specific URLs to avoid including the cThe service installer now installs a `systemd` startup script instead of an `init.d` startup script. It is up to the user to uninstall any existing `init.d` script when upgrading.+=== SolrCloud request routing++HTTP requests to SolrCloud that are for a specific core must be delivered to the node with that core, or else an HTTP 404 Not Found response will occur.+Previously, SolrCloud would try too hard scanning the cluster's state to look for it and internally route/proxy it.+If only one node is exposed to a client, and if the client uses the bin/solr export tool, it probably won't work.+=== Deprecation removals* The `jaegertracer-configurator` module, which was deprecated in 9.2, is removed. Users should migrate to the `opentelemetry` module.diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZk2TestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZk2TestBase.javaindex 513596533ff..1ac793e852d 100644--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZk2TestBase.java+++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZk2TestBase.java@@ -204,25 +204,13 @@ public abstract class AbstractBasicDistributedZk2TestBase extends AbstractFullDiSolrQuery query = new SolrQuery("*:*");- String collectionUrl = baseUrl + "/onenodecollection" + "core";- try (SolrClient client = getHttpSolrClient(baseUrl, "onenodecollectioncore")) {-- // it might take a moment for the proxy node to see us in their cloud state- waitForNon403or404or503(client, collectionUrl);-+ try (SolrClient client = getHttpSolrClient(baseUrl, "onenodecollection")) {// add a doc- SolrInputDocument doc = new SolrInputDocument();- doc.addField("id", docs);- client.add(doc);+ client.add(sdoc("id", docs));client.commit();QueryResponse results = client.query(query);assertEquals(docs - 1, results.getResults().getNumFound());- }-- try (SolrClient client = getHttpSolrClient(baseUrl, "onenodecollection")) {- QueryResponse results = client.query(query);- assertEquals(docs - 1, results.getResults().getNumFound());SolrInputDocument doc = new SolrInputDocument();doc.addField("id", docs + 1);