Apache Solr open-source search software
SOLR-9378: Internal shard requests no longer include the wasteful shard.url param.
Chris Hostetter7a55ce3852f0
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 661e1057451..26ea96a2544 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -141,6 +141,9 @@ Improvements* SOLR-16687: Add support of SolrClassLoader to SolrZkClient (Lamine Idjeraoui via Jason Gerlowski & Houston Putman)+* SOLR-9378: Internal shard requests no longer include the wasteful shard.url param. [shard] transformer now defaults to returning+ only the shard id (based on luceneMatchVersion), but can be configured to return the legacy list of replicas. (hossman)+Optimizations---------------------diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.javaindex 7135b599a8e..0ad76294335 100644--- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java@@ -367,17 +367,7 @@ public class HttpShardHandler extends ShardHandler {}private static String createSliceShardsStr(final List<String> shardUrls) {- final StringBuilder sliceShardsStr = new StringBuilder();- boolean first = true;- for (String shardUrl : shardUrls) {- if (first) {- first = false;- } else {- sliceShardsStr.append('|');- }- sliceShardsStr.append(shardUrl);- }- return sliceShardsStr.toString();+ return String.join("|", shardUrls);}private boolean canShortCircuit(diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.javaindex fb84bb827de..73cb880a7fc 100644--- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java@@ -498,7 +498,7 @@ public class SearchHandler extends RequestHandlerBase// TODO: map from shard to address[]for (String shard : sreq.actualShards) {ModifiableSolrParams params = new ModifiableSolrParams(sreq.params);- params.setShardAttributesToParams(sreq.purpose, shard);+ params.setShardAttributesToParams(sreq.purpose);// Distributed request -- need to send queryID as a part of the distributed requestparams.setNonNull(ShardParams.QUERY_ID, rb.queryID);diff --git a/solr/core/src/java/org/apache/solr/handler/component/TaskManagementHandler.java b/solr/core/src/java/org/apache/solr/handler/component/TaskManagementHandler.javaindex 43b3cdf0e9b..8986bd86cbc 100644--- a/solr/core/src/java/org/apache/solr/handler/component/TaskManagementHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/component/TaskManagementHandler.java@@ -79,7 +79,7 @@ public abstract class TaskManagementHandler extends RequestHandlerBaseString reqPath = (String) req.getContext().get(PATH);params.set(CommonParams.QT, reqPath);- params.setShardAttributesToParams(sreq.purpose, shard);+ params.setShardAttributesToParams(sreq.purpose);if (extraParams != null) {for (Map.Entry<String, String> entry : extraParams.entrySet()) {diff --git a/solr/core/src/java/org/apache/solr/response/transform/ShardAugmenterFactory.java b/solr/core/src/java/org/apache/solr/response/transform/ShardAugmenterFactory.javaindex 8cb9c5330d7..92066ba4204 100644--- a/solr/core/src/java/org/apache/solr/response/transform/ShardAugmenterFactory.java+++ b/solr/core/src/java/org/apache/solr/response/transform/ShardAugmenterFactory.java@@ -16,24 +16,85 @@*/package org.apache.solr.response.transform;-import org.apache.solr.common.params.ShardParams;+import java.util.Locale;+import java.util.Optional;+import java.util.stream.Collectors;+import org.apache.lucene.util.Version;+import org.apache.solr.cloud.CloudDescriptor;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.SolrException.ErrorCode;+import org.apache.solr.common.cloud.Replica;+import org.apache.solr.common.cloud.Slice;import org.apache.solr.common.params.SolrParams;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.core.SolrCore;import org.apache.solr.request.SolrQueryRequest;/*** @since solr 4.0*/public class ShardAugmenterFactory extends TransformerFactory {+ protected enum Style {+ URLS,+ ID;++ public static Optional<Style> getStyle(final String s) {+ if (null == s || s.trim().isEmpty()) {+ return Optional.empty();+ }+ try {+ return Optional.of(Style.valueOf(s.toUpperCase(Locale.ROOT)));+ } catch (Exception e) {+ throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown [shard] style: " + s);+ }+ }+ }++ protected Optional<Style> configuredDefaultStyle = Optional.empty();++ @Override+ public void init(NamedList<?> args) {+ super.init(args);+ if (defaultUserArgs != null) {+ configuredDefaultStyle = Style.getStyle(defaultUserArgs);+ }+ }+@Overridepublic DocTransformer create(String field, SolrParams params, SolrQueryRequest req) {- String v = req.getParams().get(ShardParams.SHARD_URL);- if (v == null) {- if (req.getParams().getBool(ShardParams.IS_SHARD, false)) {- v = "[unknown]";- } else {- v = "[not a shard request]";- }++ final SolrCore core = req.getCore();+ if (!core.getCoreContainer().isZooKeeperAware()) {+ return new ValueAugmenterFactory.ValueAugmenter(field, "[not a shard request]");+ }++ final CloudDescriptor cd = core.getCoreDescriptor().getCloudDescriptor();+ if (null == cd) { // Not sure if these can even happen?+ return new ValueAugmenterFactory.ValueAugmenter(field, "[unknown]");}- return new ValueAugmenterFactory.ValueAugmenter(field, v);++ final Style style =+ Style.getStyle(params.get("style", ""))+ .or(() -> configuredDefaultStyle)+ .orElse( // if not cnfigured at init, default Style driven by luceneMatchVersion+ core.getSolrConfig().luceneMatchVersion.onOrAfter(Version.LUCENE_9_5_0)+ ? Style.ID+ : Style.URLS);++ final String shardId = cd.getShardId();+ if (style == Style.ID) {+ return new ValueAugmenterFactory.ValueAugmenter(field, shardId);+ }++ final Slice slice =+ req.getCore()+ .getCoreContainer()+ .getZkController()+ .getClusterState()+ .getCollection(cd.getCollectionName())+ .getSlice(shardId);+ final String urls =+ slice.getReplicas().stream().map(Replica::getCoreUrl).collect(Collectors.joining("|"));+ return new ValueAugmenterFactory.ValueAugmenter(field, urls);}}diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.javaindex 90994febb24..d52525f5111 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.java@@ -199,20 +199,19 @@ public class TestCloudDeleteByQuery extends SolrCloudTestCase {assertEquals(2, docs.getNumFound());assertEquals(2, docs.size());for (SolrDocument doc : docs) {- String expected = COLLECTION_NAME + "_" + doc.getFirstValue("expected_shard_s") + "_replica";+ String expected = doc.getFirstValue("expected_shard_s").toString();String docShard = doc.getFirstValue("[shard]").toString();assertTrue("shard routing prefixes don't seem to be aligned anymore, "+ "did someone change the default routing rules? "- + "and/or the the default core name rules? "+ + "and/or the the default shard name rules? "+ "and/or the numShards used by this test? ... "- + "couldn't find "+ expected- + " as substring of [shard] == '"+ + " is ot the same as [shard] == '"+ docShard+ "' ... for docId == "+ doc.getFirstValue("id"),- docShard.contains(expected));+ docShard.equals(expected));}}diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudPseudoReturnFields.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudPseudoReturnFields.javaindex 3db1166fae0..434bd380a14 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudPseudoReturnFields.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudPseudoReturnFields.java@@ -16,16 +16,21 @@*/package org.apache.solr.cloud;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.startsWith;+import java.lang.invoke.MethodHandles;import java.nio.file.Path;import java.util.ArrayList;import java.util.Arrays;import java.util.Collections;import java.util.HashMap;+import java.util.HashSet;import java.util.List;import java.util.Locale;import java.util.Map;import java.util.Random;+import java.util.Set;import org.apache.lucene.tests.util.TestUtil;import org.apache.solr.client.solrj.SolrClient;import org.apache.solr.client.solrj.SolrServerException;@@ -42,6 +47,7 @@ import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.params.SolrParams;import org.apache.solr.embedded.JettySolrRunner;import org.apache.solr.search.TestPseudoReturnFields;+import org.hamcrest.MatcherAssert;import org.junit.AfterClass;import org.junit.Before;import org.junit.BeforeClass;@@ -56,6 +62,9 @@ public class TestCloudPseudoReturnFields extends SolrCloudTestCase {private static final String DEBUG_LABEL = MethodHandles.lookup().lookupClass().getName();private static final String COLLECTION_NAME = DEBUG_LABEL + "_collection";+ // randomized for testing '[shards]' behavior...+ private static int repFactor;+/** A collection specific client for operations at the cloud level */private static CloudSolrClient COLLECTION_CLIENT;/** One client per node */@@ -63,9 +72,10 @@ public class TestCloudPseudoReturnFields extends SolrCloudTestCase {@BeforeClasspublic static void createMiniSolrCloudCluster() throws Exception {- // multi replicas should matter...- final int repFactor = usually() ? 1 : 2;- // ... but we definitely want to ensure forwarded requests to other shards work ...+ // replication factor will impact wether we expect a list of urls from the '[shard]'+ // augmenter...+ repFactor = usually() ? 1 : 2;+ // ... and we definitely want to ensure forwarded requests to other shards work ...final int numShards = 2;// ... including some forwarded requests from nodes not hosting a shardfinal int numNodes = 1 + (numShards * repFactor);@@ -698,9 +708,18 @@ public class TestCloudPseudoReturnFields extends SolrCloudTestCase {for (SolrParams p :Arrays.asList(- params("q", "*:*", "fl", "[docid],[shard],[explain],x_alias:[value v=10 t=int]"),params(- "q", "*:*", "fl", "[docid],[shard]", "fl", "[explain],x_alias:[value v=10 t=int]"),+ "q",+ "*:*",+ "fl",+ "[docid],[shard],replica_urls:[shard style='urls'],shard_id:[shard style='id'],[explain],x_alias:[value v=10 t=int]"),+ params(+ "q",+ "*:*",+ "fl",+ "[docid],[shard]",+ "fl",+ "replica_urls:[shard style='urls'],shard_id:[shard style='id'],[explain],x_alias:[value v=10 t=int]"),params("q","*:*",@@ -709,6 +728,10 @@ public class TestCloudPseudoReturnFields extends SolrCloudTestCase {"fl","[shard]","fl",+ "replica_urls:[shard style='urls']",+ "fl",+ "shard_id:[shard style='id']",+ "fl","[explain]","fl","x_alias:[value v=10 t=int]"))) {@@ -717,14 +740,74 @@ public class TestCloudPseudoReturnFields extends SolrCloudTestCase {// shouldn't matter what doc we pick...for (SolrDocument doc : docs) {String msg = p + " => " + doc;- assertEquals(msg, 4, doc.size());+ assertEquals(msg, 6, doc.size());assertTrue(msg, doc.getFieldValue("[docid]") instanceof Integer);- assertTrue(msg, doc.getFieldValue("[shard]") instanceof String);++ assertTrue(msg, doc.getFieldValue("shard_id") instanceof String);+ MatcherAssert.assertThat(doc.getFieldValue("shard_id").toString(), startsWith("shard"));++ assertTrue(msg, doc.getFieldValue("replica_urls") instanceof String);+ MatcherAssert.assertThat(+ doc.getFieldValue("replica_urls").toString(),+ containsString(+ "/solr/org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard"));+ if (1 < repFactor) {+ MatcherAssert.assertThat(+ doc.getFieldValue("replica_urls").toString(), containsString("|"));+ }++ assertEquals(msg, doc.getFieldValue("shard_id"), doc.getFieldValue("[shard]"));+assertTrue(msg, doc.getFieldValue("[explain]") instanceof String);assertTrue(msg, doc.getFieldValue("x_alias") instanceof Integer);assertEquals(msg, 10, doc.getFieldValue("x_alias"));}}++ // [shard] should still work with routing options+ for (SolrParams p :+ Arrays.asList(+ params(+ "q",+ "*:*",+ "_route_",+ "blah!", // doesn't matter, just forcing a single shard+ "fl",+ "id,[shard],replica_urls:[shard style='urls'],shard_id:[shard style='id']"),+ params(+ "q",+ "*:*",+ "shards",+ "shard1", // doesn't matter, just forcing a single shard+ "fl",+ "id,[shard],replica_urls:[shard style='urls'],shard_id:[shard style='id']"))) {+ docs = assertSearch(p);+ // Don't make assumptions about exact shard distribution, just assert we got at least one doc+ assertTrue(+ "Not enough docs in shard -- did routing rules change? => " + docs, 1 <= docs.size());+ Set<Object> shardsSeen = new HashSet<>();+ for (SolrDocument doc : docs) {+ String msg = p + " => " + doc;++ assertTrue(msg, doc.getFieldValue("shard_id") instanceof String);+ MatcherAssert.assertThat(doc.getFieldValue("shard_id").toString(), startsWith("shard"));++ assertTrue(msg, doc.getFieldValue("replica_urls") instanceof String);+ MatcherAssert.assertThat(+ doc.getFieldValue("replica_urls").toString(),+ containsString(+ "/solr/org.apache.solr.cloud.TestCloudPseudoReturnFields_collection_shard"));+ if (1 < repFactor) {+ MatcherAssert.assertThat(+ doc.getFieldValue("replica_urls").toString(), containsString("|"));+ }++ assertEquals(msg, doc.getFieldValue("shard_id"), doc.getFieldValue("[shard]"));++ shardsSeen.add(doc.getFieldValue("shard_id"));+ }+ assertEquals("Only expected results from one shard => " + docs, 1, shardsSeen.size());+ }}public void testDocIdAugmenterRTG() throws Exception {diff --git a/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorCloud.java b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorCloud.javaindex 79f1f1b78cc..612fdb1615e 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorCloud.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorCloud.java@@ -188,20 +188,19 @@ public class TestTolerantUpdateProcessorCloud extends SolrCloudTestCase {assertEquals(2, docs.getNumFound());assertEquals(2, docs.size());for (SolrDocument doc : docs) {- String expected = COLLECTION_NAME + "_" + doc.getFirstValue("expected_shard_s") + "_replica";+ String expected = doc.getFirstValue("expected_shard_s").toString();String docShard = doc.getFirstValue("[shard]").toString();assertTrue("shard routing prefixes don't seem to be aligned anymore, "+ "did someone change the default routing rules? "- + "and/or the the default core name rules? "+ + "and/or the the default shard name rules? "+ "and/or the numShards used by this test? ... "- + "couldn't find "+ expected- + " as substring of [shard] == '"+ + " is ot the same as [shard] == '"+ docShard+ "' ... for docId == "+ doc.getFirstValue("id"),- docShard.contains(expected));+ docShard.equals(expected));}}diff --git a/solr/core/src/test/org/apache/solr/handler/component/ShardsAllowListTest.java b/solr/core/src/test/org/apache/solr/handler/component/ShardsAllowListTest.javaindex 6a30ba88db4..174ebbd7a8d 100644--- a/solr/core/src/test/org/apache/solr/handler/component/ShardsAllowListTest.java+++ b/solr/core/src/test/org/apache/solr/handler/component/ShardsAllowListTest.java@@ -250,8 +250,6 @@ public class ShardsAllowListTest extends MultiSolrCloudTestCase {implicitCluster,"distrib","false",- "shard.url",- getShardUrl("shard2", explicitCluster),"shards.purpose","64","isShard",diff --git a/solr/solr-ref-guide/modules/query-guide/pages/document-transformers.adoc b/solr/solr-ref-guide/modules/query-guide/pages/document-transformers.adocindex c1ddce4ed91..3821f772a94 100644--- a/solr/solr-ref-guide/modules/query-guide/pages/document-transformers.adoc+++ b/solr/solr-ref-guide/modules/query-guide/pages/document-transformers.adoc@@ -212,8 +212,20 @@ When paths do not start with a `/` they are treated as "path suffixes":This transformer adds information about what shard each individual document came from in a distributed request.-ShardAugmenterFactory does not support any request parameters, or configuration options.+[source,plain]+----+q=fl=id,[shard]+----++A `style` local parameter can be specified as either `id` (the default) or `urls` which returns a pipe delimited list of all replica urls for backcompat reasons.+The default style can be changed by specifying an `args` parameter in your `solrconfig.xml` configuration:+[source,xml]+----+<transformer name="shard" class="org.apache.solr.response.transform.ShardAugmenterFactory" >+ <str name="args">urls</str>+</transformer>+----=== [docid] - DocIdAugmenterFactorydiff --git a/solr/solrj/src/java/org/apache/solr/common/params/ModifiableSolrParams.java b/solr/solrj/src/java/org/apache/solr/common/params/ModifiableSolrParams.javaindex 47469e2e911..331773660ed 100644--- a/solr/solrj/src/java/org/apache/solr/common/params/ModifiableSolrParams.java+++ b/solr/solrj/src/java/org/apache/solr/common/params/ModifiableSolrParams.java@@ -178,14 +178,13 @@ public class ModifiableSolrParams extends SolrParams {// ----------------------------------------------------------------// ----------------------------------------------------------------- public void setShardAttributesToParams(int purpose, String shardUrl) {+ public void setShardAttributesToParams(int purpose) {remove(ShardParams.SHARDS); // not a top-level requestset(DISTRIB, Boolean.FALSE.toString()); // not a top-level requestremove(INDENT);remove(CommonParams.HEADER_ECHO_PARAMS);set(ShardParams.IS_SHARD, true); // a sub (shard) requestset(ShardParams.SHARDS_PURPOSE, purpose);- set(ShardParams.SHARD_URL, shardUrl); // so the shard knows what was askedset(CommonParams.OMIT_HEADER, false);}diff --git a/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java b/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.javaindex fd423ec5085..8a6b87577ec 100644--- a/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java+++ b/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java@@ -42,8 +42,13 @@ public interface ShardParams {/** whether the request goes to a shard */String IS_SHARD = "isShard";- /** The requested URL for this shard */- String SHARD_URL = "shard.url";+ /**+ * The requested URL for this shard+ *+ * @deprecated This was an internally used param never ment for clients to specify; it is no+ * longer used by Solr.+ */+ @Deprecated String SHARD_URL = "shard.url";/** The requested shard name */String SHARD_NAME = "shard.name";diff --git a/solr/solrj/src/test/org/apache/solr/common/params/ModifiableSolrParamsTest.java b/solr/solrj/src/test/org/apache/solr/common/params/ModifiableSolrParamsTest.javaindex 91871062889..9cf97fe1cdf 100644--- a/solr/solrj/src/test/org/apache/solr/common/params/ModifiableSolrParamsTest.java+++ b/solr/solrj/src/test/org/apache/solr/common/params/ModifiableSolrParamsTest.java@@ -128,17 +128,15 @@ public class ModifiableSolrParamsTest extends SolrTestCase {}public void testSetShardAttributesToParams() {- var dummyShardUrl = "Shard-Url";var dummyIndent = "Dummy-Indent";modifiable.set(ShardParams.SHARDS, "dummyValue");modifiable.set(CommonParams.HEADER_ECHO_PARAMS, "dummyValue");modifiable.set(CommonParams.INDENT, dummyIndent);- modifiable.setShardAttributesToParams(2, dummyShardUrl);+ modifiable.setShardAttributesToParams(2);assertEquals(Boolean.FALSE.toString(), modifiable.get(CommonParams.DISTRIB));- assertEquals(dummyShardUrl, modifiable.get(ShardParams.SHARD_URL));assertEquals("2", modifiable.get(ShardParams.SHARDS_PURPOSE));assertEquals(Boolean.FALSE.toString(), modifiable.get(CommonParams.OMIT_HEADER));assertEquals(Boolean.TRUE.toString(), modifiable.get(ShardParams.IS_SHARD));diff --git a/solr/solrj/src/test/org/apache/solr/common/params/ShardParamsTest.java b/solr/solrj/src/test/org/apache/solr/common/params/ShardParamsTest.javaindex e1c926638d8..63f09d644d3 100644--- a/solr/solrj/src/test/org/apache/solr/common/params/ShardParamsTest.java+++ b/solr/solrj/src/test/org/apache/solr/common/params/ShardParamsTest.java@@ -45,10 +45,6 @@ public class ShardParamsTest extends SolrTestCase {assertEquals(ShardParams.IS_SHARD, "isShard");}- public void testShardUrl() {- assertEquals(ShardParams.SHARD_URL, "shard.url");- }-public void testShardsQt() {assertEquals(ShardParams.SHARDS_QT, "shards.qt");}