Apache Solr open-source search software
SOLR-16933: Include full query response in API tool (#1863)
Houston Putmanf5b0d22f0636
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex e5e59103585..009f9bc2d44 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -138,6 +138,8 @@ Bug Fixes* SOLR-16946: Updated Cluster Singleton plugins are stopped correctly when the Overseer is closed. (Paul McArthur)+* SOLR-16933: Include the full query response when using the API tool, and fix serialization issues for SolrDocumentList. (Houston Putman)+Dependency Upgrades---------------------diff --git a/solr/core/src/java/org/apache/solr/cli/ApiTool.java b/solr/core/src/java/org/apache/solr/cli/ApiTool.javaindex 6122000211b..82980ea5257 100644--- a/solr/core/src/java/org/apache/solr/cli/ApiTool.java+++ b/solr/core/src/java/org/apache/solr/cli/ApiTool.java@@ -23,11 +23,10 @@ import java.util.List;import org.apache.commons.cli.CommandLine;import org.apache.commons.cli.Option;import org.apache.solr.client.solrj.SolrRequest;+import org.apache.solr.client.solrj.impl.NoOpResponseParser;import org.apache.solr.client.solrj.request.GenericSolrRequest;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.util.NamedList;-import org.noggit.CharArr;-import org.noggit.JSONWriter;/*** Supports api command in the bin/solr script.@@ -61,29 +60,39 @@ public class ApiTool extends ToolBase {@Overridepublic void runImpl(CommandLine cli) throws Exception {+ String response = null;String getUrl = cli.getOptionValue("get");if (getUrl != null) {- getUrl = getUrl.replace("+", "%20");- URI uri = new URI(getUrl);- String solrUrl = getSolrUrlFromUri(uri);- String path = uri.getPath();- try (var solrClient = SolrCLI.getSolrClient(solrUrl)) {- NamedList<Object> response =- solrClient.request(- // For path parameter we need the path without the root so from the second / char- // (because root can be configured)- // E.g URL is http://localhost:8983/solr/admin/info/system path is- // /solr/admin/info/system and the path without root is /admin/info/system- new GenericSolrRequest(- SolrRequest.METHOD.GET,- path.substring(path.indexOf("/", path.indexOf("/") + 1)),- getSolrParamsFromUri(uri)));+ response = callGet(getUrl);+ }+ if (response != null) {+ // pretty-print the response to stdout+ echo(response);+ }+ }- // pretty-print the response to stdout- CharArr arr = new CharArr();- new JSONWriter(arr, 2).write(response.asMap());- echo(arr.toString());- }+ protected String callGet(String url) throws Exception {+ URI uri = new URI(url.replace("+", "%20"));+ String solrUrl = getSolrUrlFromUri(uri);+ String path = uri.getPath();+ try (var solrClient = SolrCLI.getSolrClient(solrUrl)) {+ // For path parameter we need the path without the root so from the second / char+ // (because root can be configured)+ // E.g URL is http://localhost:8983/solr/admin/info/system path is+ // /solr/admin/info/system and the path without root is /admin/info/system+ var req =+ new GenericSolrRequest(+ SolrRequest.METHOD.GET,+ path.substring(path.indexOf("/", path.indexOf("/") + 1)),+ getSolrParamsFromUri(uri) // .add("indent", "true")+ );+ // Using the "smart" solr parsers won't work, because they decode into Solr objects.+ // When trying to re-write into JSON, the JSONWriter doesn't have the right info to print it+ // correctly.+ // All we want to do is pass the JSON response to the user, so do that.+ req.setResponseParser(new NoOpResponseParser("json"));+ NamedList<Object> response = solrClient.request(req);+ return (String) response.get("response");}}diff --git a/solr/core/src/java/org/apache/solr/cli/CreateTool.java b/solr/core/src/java/org/apache/solr/cli/CreateTool.javaindex 91a2214eeb4..cd7aaddb1f1 100644--- a/solr/core/src/java/org/apache/solr/cli/CreateTool.java+++ b/solr/core/src/java/org/apache/solr/cli/CreateTool.java@@ -38,6 +38,7 @@ import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;import org.apache.solr.client.solrj.impl.CloudSolrClient;import org.apache.solr.client.solrj.impl.Http2SolrClient;+import org.apache.solr.client.solrj.impl.NoOpResponseParser;import org.apache.solr.client.solrj.request.CollectionAdminRequest;import org.apache.solr.client.solrj.request.CoreAdminRequest;import org.apache.solr.client.solrj.request.GenericSolrRequest;@@ -48,8 +49,6 @@ import org.apache.solr.common.params.CollectionAdminParams;import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.core.ConfigSetService;-import org.noggit.CharArr;-import org.noggit.JSONWriter;/** Supports create command in the bin/solr script. */public class CreateTool extends ToolBase {@@ -287,19 +286,18 @@ public class CreateTool extends ToolBase {NamedList<Object> response;try {- response =- cloudSolrClient.request(- CollectionAdminRequest.createCollection(- collectionName, confName, numShards, replicationFactor));+ var req =+ CollectionAdminRequest.createCollection(+ collectionName, confName, numShards, replicationFactor);+ req.setResponseParser(new NoOpResponseParser("json"));+ response = cloudSolrClient.request(req);} catch (SolrServerException sse) {throw new Exception("Failed to create collection '" + collectionName + "' due to: " + sse.getMessage());}if (cli.hasOption(SolrCLI.OPTION_VERBOSE.getOpt())) {- CharArr arr = new CharArr();- new JSONWriter(arr, 2).write(response.asMap());- echo(arr.toString());+ echo((String) response.get("response"));} else {String endMessage =String.format(diff --git a/solr/core/src/java/org/apache/solr/cli/DeleteTool.java b/solr/core/src/java/org/apache/solr/cli/DeleteTool.javaindex 5e556f02017..1b8eec4e7bf 100644--- a/solr/core/src/java/org/apache/solr/cli/DeleteTool.java+++ b/solr/core/src/java/org/apache/solr/cli/DeleteTool.java@@ -32,12 +32,11 @@ import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;import org.apache.solr.client.solrj.impl.CloudSolrClient;import org.apache.solr.client.solrj.impl.Http2SolrClient;+import org.apache.solr.client.solrj.impl.NoOpResponseParser;import org.apache.solr.client.solrj.request.CollectionAdminRequest;import org.apache.solr.client.solrj.request.CoreAdminRequest;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.util.NamedList;-import org.noggit.CharArr;-import org.noggit.JSONWriter;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -173,7 +172,9 @@ public class DeleteTool extends ToolBase {NamedList<Object> response;try {- response = cloudSolrClient.request(CollectionAdminRequest.deleteCollection(collectionName));+ var req = CollectionAdminRequest.deleteCollection(collectionName);+ req.setResponseParser(new NoOpResponseParser("json"));+ response = cloudSolrClient.request(req);} catch (SolrServerException sse) {throw new Exception("Failed to delete collection '" + collectionName + "' due to: " + sse.getMessage());@@ -194,9 +195,7 @@ public class DeleteTool extends ToolBase {}if (response != null) {- CharArr arr = new CharArr();- new JSONWriter(arr, 2).write(response.asMap());- echo(arr.toString());+ echo((String) response.get("response"));echo("\n");}@@ -215,15 +214,14 @@ public class DeleteTool extends ToolBase {unloadRequest.setDeleteDataDir(true);unloadRequest.setDeleteInstanceDir(true);unloadRequest.setCoreName(coreName);+ unloadRequest.setResponseParser(new NoOpResponseParser("json"));response = solrClient.request(unloadRequest);} catch (SolrServerException sse) {throw new Exception("Failed to delete core '" + coreName + "' due to: " + sse.getMessage());}if (response != null) {- CharArr arr = new CharArr();- new JSONWriter(arr, 2).write(response.asMap());- echoIfVerbose(arr.toString(), cli);+ echoIfVerbose((String) response.get("response"), cli);echoIfVerbose("\n", cli);}}diff --git a/solr/core/src/test/org/apache/solr/cli/ApiToolTest.java b/solr/core/src/test/org/apache/solr/cli/ApiToolTest.javaindex 82e9b99a473..d817845854a 100644--- a/solr/core/src/test/org/apache/solr/cli/ApiToolTest.java+++ b/solr/core/src/test/org/apache/solr/cli/ApiToolTest.java@@ -16,14 +16,30 @@*/package org.apache.solr.cli;+import java.io.File;import java.net.URI;import java.net.URISyntaxException;import java.util.Arrays;-import org.apache.solr.SolrTestCase;+import java.util.Locale;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.solr.client.solrj.request.AbstractUpdateRequest;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.request.UpdateRequest;+import org.apache.solr.cloud.SolrCloudTestCase;import org.apache.solr.common.params.ModifiableSolrParams;+import org.junit.BeforeClass;import org.junit.Test;-public class ApiToolTest extends SolrTestCase {+public class ApiToolTest extends SolrCloudTestCase {+ static String COLLECTION_NAME = "globalLoaderColl";++ @BeforeClass+ public static void setupCluster() throws Exception {+ configureCluster(1)+ .addConfig(+ "config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))+ .configure();+ }@Testpublic void testParsingGetUrl() throws URISyntaxException {@@ -42,4 +58,48 @@ public class ApiToolTest extends SolrTestCase {assertEquals("select id from COLL_NAME limit 10", params.get("stmt"));}}++ @Test+ public void testQueryResponse() throws Exception {+ int docCount = 1000;+ CollectionAdminRequest.createCollection(COLLECTION_NAME, "config", 2, 1)+ .process(cluster.getSolrClient());+ cluster.waitForActiveCollection(COLLECTION_NAME, 2, 2);++ String tmpFileLoc =+ new File(cluster.getBaseDir().toFile().getAbsolutePath() + File.separator).getPath();++ UpdateRequest ur = new UpdateRequest();+ ur.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);++ for (int i = 0; i < docCount; i++) {+ ur.add(+ "id",+ String.valueOf(i),+ "desc_s",+ TestUtil.randomSimpleString(random(), 10, 50),+ "a_dt",+ "2019-09-30T05:58:03Z");+ }+ cluster.getSolrClient().request(ur, COLLECTION_NAME);++ ApiTool tool = new ApiTool();++ String response =+ tool.callGet(+ cluster.getJettySolrRunner(0).getBaseUrl()+ + "/"+ + COLLECTION_NAME+ + "/select?q=*:*&rows=1&fl=id&sort=id+asc");+ // Fields that could be missed because of serialization+ assertFindInJson(response, "\"numFound\":1000,");+ // Correct formatting+ assertFindInJson(response, "\"docs\":[{");+ }++ private void assertFindInJson(String json, String find) {+ assertTrue(+ String.format(Locale.ROOT, "Could not find string %s in response: \n%s", find, json),+ json.contains(find));+ }}diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/NodeValueFetcher.java b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/NodeValueFetcher.javaindex c5832aa1432..c6e122aae50 100644--- a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/NodeValueFetcher.java+++ b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/NodeValueFetcher.java@@ -139,7 +139,7 @@ public class NodeValueFetcher {try {SimpleSolrResponse rsp = ctx.invokeWithRetry(solrNode, CommonParams.METRICS_PATH, params);- NamedList<?> metrics = (NamedList<?>) rsp.nl.get("metrics");+ NamedList<?> metrics = (NamedList<?>) rsp.getResponse().get("metrics");if (metrics != null) {for (Tags t : Tags.values()) {ctx.tags.put(t.tagName, t.extractResult(metrics));diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.javaindex 1bc2aed24bc..4d7f087426a 100644--- a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java+++ b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java@@ -194,7 +194,8 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriterSimpleSolrResponse rsp = ctx.invokeWithRetry(solrNode, CommonParams.METRICS_PATH, params);metricsKeyVsTag.forEach((key, tags) -> {- Object v = Utils.getObjectByPath(rsp.nl, true, Arrays.asList("metrics", key));+ Object v =+ Utils.getObjectByPath(rsp.getResponse(), true, Arrays.asList("metrics", key));for (Object tag : tags) {if (tag instanceof Function) {@SuppressWarnings({"unchecked"})@@ -296,7 +297,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter.withResponseParser(new BinaryResponseParser()).build()) {NamedList<Object> rsp = client.request(request);- request.response.nl = rsp;+ request.response.setResponse(rsp);return request.response;}}diff --git a/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java b/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.javaindex f321646a5c6..1c2274b7c0f 100644--- a/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java+++ b/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java@@ -104,7 +104,7 @@ public interface IteratorWriter extends JSONWriter.Writable {writer.writeValueSeparator();}writer.indent();- write(writer);+ writer.write(o);return this;}});diff --git a/solr/solrj/src/java/org/apache/solr/common/SolrDocumentList.java b/solr/solrj/src/java/org/apache/solr/common/SolrDocumentList.javaindex 4204fc2e61b..cbc7f8dd2b5 100644--- a/solr/solrj/src/java/org/apache/solr/common/SolrDocumentList.java+++ b/solr/solrj/src/java/org/apache/solr/common/SolrDocumentList.java@@ -16,7 +16,9 @@*/package org.apache.solr.common;+import java.io.IOException;import java.util.ArrayList;+import java.util.Iterator;/*** Represent a list of SolrDocuments returned from a search. This includes position and offset@@ -24,7 +26,24 @@ import java.util.ArrayList;** @since solr 1.3*/-public class SolrDocumentList extends ArrayList<SolrDocument> {+public class SolrDocumentList extends ArrayList<SolrDocument> implements MapWriter {++ @Override+ public void writeMap(EntryWriter ew) throws IOException {+ ew.put("numFound", numFound);+ ew.put("start", start);++ if (maxScore != null) {+ ew.put("maxScore", maxScore);+ }++ if (numFoundExact != null) {+ ew.put("numFoundExact", numFoundExact);+ }+ final Iterator<SolrDocument> docs = iterator();+ ew.put("docs", (IteratorWriter) iw -> docs.forEachRemaining(iw::addNoEx));+ }+private long numFound = 0;private long start = 0;private Float maxScore = null;diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.javaindex 391ad2eaed2..31d6e736a2e 100644--- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java+++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java@@ -479,15 +479,19 @@ public class Utils {Object o = getVal(obj, s, -1);if (o == null) return null;if (idx > -1) {- if (o instanceof MapWriter) {+ if (o instanceof List) {+ List<?> l = (List<?>) o;+ o = idx < l.size() ? l.get(idx) : null;+ } else if (o instanceof IteratorWriter) {+ o = getValueAt((IteratorWriter) o, idx);+ } else if (o instanceof MapWriter) {o = getVal(o, null, idx);} else if (o instanceof Map) {@SuppressWarnings("unchecked")Map<String, Object> map = (Map<String, Object>) o;o = getVal(new MapWriterMap(map), null, idx);} else {- List<?> l = (List<?>) o;- o = idx < l.size() ? l.get(idx) : null;+ return null;}}if (!isMapLike(o)) return null;diff --git a/solr/solrj/src/java/org/noggit/JSONWriter.java b/solr/solrj/src/java/org/noggit/JSONWriter.javaindex 0782717b8ae..349c1fecf02 100644--- a/solr/solrj/src/java/org/noggit/JSONWriter.java+++ b/solr/solrj/src/java/org/noggit/JSONWriter.java@@ -21,6 +21,7 @@ package org.noggit;import java.util.Arrays;import java.util.Collection;+import java.util.Iterator;import java.util.Map;public class JSONWriter {@@ -70,6 +71,8 @@ public class JSONWriter {// perfect hashing.if (o == null) {writeNull();+ } else if (o instanceof Writable) {+ ((Writable) o).write(this);} else if (o instanceof String) {writeString((String) o);} else if (o instanceof Number) {@@ -86,12 +89,12 @@ public class JSONWriter {write((Map<?, ?>) o);} else if (o instanceof Collection) {write((Collection<?>) o);+ } else if (o instanceof Iterator) {+ write((Iterator<?>) o);} else if (o instanceof Boolean) {write(((Boolean) o).booleanValue());} else if (o instanceof CharSequence) {writeString((CharSequence) o);- } else if (o instanceof Writable) {- ((Writable) o).write(this);} else if (o instanceof Object[]) {write(Arrays.asList((Object[]) o));} else if (o instanceof int[]) {@@ -156,6 +159,20 @@ public class JSONWriter {endArray();}+ public void write(Iterator<?> val) {+ startArray();+ boolean first = true;+ while (val.hasNext()) {+ if (first) {+ first = false;+ } else {+ writeValueSeparator();+ }+ write(val.next());+ }+ endArray();+ }+/*** A byte[] may be either a single logical value, or a list of small integers. It's up to the* implementation to decide.