Apache Solr open-source search software
SOLR-18513: Port ExportTool to picocli (#5040)
Serhiy Bzhezytskyy162db9ec3920
diff --git a/changelog/unreleased/SOLR-17697-picocli-experimental-cli.yml b/changelog/unreleased/SOLR-17697-picocli-experimental-cli.ymlindex c008021477d..3510707f7e0 100644--- a/changelog/unreleased/SOLR-17697-picocli-experimental-cli.yml+++ b/changelog/unreleased/SOLR-17697-picocli-experimental-cli.yml@@ -19,3 +19,5 @@ links:url: https://issues.apache.org/jira/browse/SOLR-18511- name: SOLR-18512url: https://issues.apache.org/jira/browse/SOLR-18512+ - name: SOLR-18513+ url: https://issues.apache.org/jira/browse/SOLR-18513diff --git a/solr/core/src/java/org/apache/solr/cli/ExportTool.java b/solr/core/src/java/org/apache/solr/cli/ExportTool.javaindex b63c2ba002d..03dacb31f48 100644--- a/solr/core/src/java/org/apache/solr/cli/ExportTool.java+++ b/solr/core/src/java/org/apache/solr/cli/ExportTool.java@@ -84,8 +84,22 @@ import org.noggit.CharArr;import org.noggit.JSONWriter;/** Supports export command in the bin/solr script. */+@SuppressWarnings("UnnecessarilyFullyQualified")+ name = "export",+ description = "Exports documents from a collection to a local file.",+ footerHeading = "%nExamples:%n",+ footer = {+ " # Export a collection's documents to gettingstarted.json",+ " bin/solr export -c gettingstarted --solr-url http://localhost:8983"+ })public class ExportTool extends ToolBase {+ /**+ * @deprecated Only used by the commons-cli parser; the picocli path declares this as an annotated+ * field.+ */+ @Deprecatedprivate static final Option COLLECTION_NAME_OPTION =Option.builder("c").longOpt("name")@@ -94,6 +108,11 @@ public class ExportTool extends ToolBase {.desc("Name of the collection.").get();+ /**+ * @deprecated Only used by the commons-cli parser; the picocli path declares this as an annotated+ * field.+ */+ @Deprecatedprivate static final Option OUTPUT_OPTION =Option.builder().longOpt("output")@@ -103,6 +122,11 @@ public class ExportTool extends ToolBase {"Path to output the exported data, and optionally the file name, defaults to 'collection-name'.").get();+ /**+ * @deprecated Only used by the commons-cli parser; the picocli path declares this as an annotated+ * field.+ */+ @Deprecatedprivate static final Option FORMAT_OPTION =Option.builder().longOpt("format")@@ -111,9 +135,19 @@ public class ExportTool extends ToolBase {.desc("Output format for exported docs (json, jsonl or javabin), defaulting to json.").get();+ /**+ * @deprecated Only used by the commons-cli parser; the picocli path declares this as an annotated+ * field.+ */+ @Deprecatedprivate static final Option COMPRESS_OPTION =Option.builder().longOpt("compress").desc("Compress the output. Defaults to false.").get();+ /**+ * @deprecated Only used by the commons-cli parser; the picocli path declares this as an annotated+ * field.+ */+ @Deprecatedprivate static final Option LIMIT_OPTION =Option.builder().longOpt("limit")@@ -122,6 +156,11 @@ public class ExportTool extends ToolBase {.desc("Maximum number of docs to download. Default is 100, use -1 for all docs.").get();+ /**+ * @deprecated Only used by the commons-cli parser; the picocli path declares this as an annotated+ * field.+ */+ @Deprecatedprivate static final Option QUERY_OPTION =Option.builder().longOpt("query")@@ -130,6 +169,11 @@ public class ExportTool extends ToolBase {.desc("A custom query, default is '*:*'.").get();+ /**+ * @deprecated Only used by the commons-cli parser; the picocli path declares this as an annotated+ * field.+ */+ @Deprecatedprivate static final Option FIELDS_OPTION =Option.builder().longOpt("fields")@@ -149,6 +193,74 @@ public class ExportTool extends ToolBase {String fields,String limit) {}+ // --- picocli fields ---+ // The connection group is mandatory (multiplicity "1"): the commons-cli path throws+ // IllegalArgumentException when no connection target is given, so ArgGroup enforces the same+ // requirement declaratively.++ @picocli.CommandLine.ArgGroup(exclusive = true, multiplicity = "1")+ private ConnectionOptions connectionOptions;++ @picocli.CommandLine.Mixin private CredentialsOptions credentialsOptions;++ @picocli.CommandLine.Option(+ names = {"-c", "--name"},+ required = true,+ paramLabel = "NAME",+ description = "Name of the collection.")+ private String nameOpt;++ @picocli.CommandLine.Option(+ names = "--output",+ paramLabel = "PATH",+ description =+ "Path to output the exported data, and optionally the file name, defaults to"+ + " 'collection-name'.")+ private String outputOpt;++ /** The values of {@code --format}; each constant is spelled as it is on the command line. */+ enum Format {+ json,+ jsonl,+ javabin+ }++ @picocli.CommandLine.Option(+ names = "--format",+ paramLabel = "FORMAT",+ description =+ "Output format for exported docs (${COMPLETION-CANDIDATES}), defaulting to json.")+ private Format formatOpt;++ @picocli.CommandLine.Option(+ names = "--compress",+ description = "Compress the output. Defaults to false.")+ private boolean compressOpt;++ @picocli.CommandLine.Option(+ names = "--limit",+ defaultValue = "100",+ paramLabel = "#",+ description = "Maximum number of docs to download. Default is 100, use -1 for all docs.")+ private long limitOpt;++ @picocli.CommandLine.Option(+ names = "--query",+ defaultValue = "*:*",+ paramLabel = "QUERY",+ description = "A custom query, default is '*:*'.")+ private String queryOpt;++ @picocli.CommandLine.Option(+ names = "--fields",+ paramLabel = "FIELDA,FIELDB",+ description = "Comma separated list of fields to export. By default all fields are fetched.")+ private String fieldsOpt;++ public ExportTool() {+ this(new DefaultToolRuntime());+ }+public ExportTool(ToolRuntime runtime) {super(runtime);}@@ -729,6 +841,28 @@ public class ExportTool extends ToolBase {@Overridepublic int callTool() throws Exception {- throw new UnsupportedOperationException("This tool does not yet support PicoCli");+ String url = resolveSolrUrl(credentialsOptions.credentials) + "/solr/" + nameOpt;+ ExportParams params =+ new ExportParams(+ url,+ credentialsOptions.credentials,+ queryOpt,+ outputOpt,+ formatOpt == null ? null : formatOpt.name(),+ compressOpt,+ fieldsOpt,+ String.valueOf(limitOpt));+ export(params);+ return 0;+ }++ private String resolveSolrUrl(String credentials) throws Exception {+ String solrUrlArg = connectionOptions.effectiveSolrUrl();+ if (solrUrlArg != null) {+ return CLIUtils.normalizeSolrUrl(solrUrlArg);+ }+ return CLIUtils.solrUrlFromConnection(+ CloudSolrClient.CloudSolrClientConnection.parse(connectionOptions.effectiveZkHost()),+ credentials);}}diff --git a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java b/solr/core/src/java/org/apache/solr/cli/SolrCLI.javaindex 93ffd2965e4..2dd48b27cad 100755--- a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java+++ b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java@@ -84,7 +84,8 @@ import org.slf4j.LoggerFactory;AssertTool.class,ApiTool.class,ClusterTool.class,- ConfigTool.class+ ConfigTool.class,+ ExportTool.class})public class SolrCLI implements CLIO, java.util.concurrent.Callable<Integer> {diff --git a/solr/core/src/test/org/apache/solr/cli/ExportToolPicocliTest.java b/solr/core/src/test/org/apache/solr/cli/ExportToolPicocliTest.javanew file mode 100644index 00000000000..62781d4c4f4--- /dev/null+++ b/solr/core/src/test/org/apache/solr/cli/ExportToolPicocliTest.java@@ -0,0 +1,46 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.cli;++import java.util.Arrays;+import org.junit.Test;+import picocli.CommandLine;++/**+ * Runs all {@link ExportToolTest} tests through the picocli invocation path.+ *+ * <p>All {@code @Test} methods are inherited; only the invocation strategy is overridden.+ */+public class ExportToolPicocliTest extends ExportToolTest {++ @Override+ protected int runTool(String[] args, Class<? extends ToolBase> clazz) throws Exception {+ // args[0] is the tool name used by commons-cli dispatch; strip it for picocli.+ String[] toolArgs = Arrays.copyOfRange(args, 1, args.length);+ ToolRuntime runtime = new CLITestHelper.TestingRuntime(false);+ ToolBase tool = clazz.getDeclaredConstructor(ToolRuntime.class).newInstance(runtime);+ return new CommandLine(tool)+ .setDefaultValueProvider(new CliDefaultValueProvider())+ .execute(toolArgs);+ }++ @Test+ public void testNonNumericLimitIsAUsageError() throws Exception {+ // picocli reads --limit as a number, so it rejects "abc" before the tool runs: usage error 2+ assertEquals(2, exportTo(createTempDir(), "--limit", "abc"));+ }+}diff --git a/solr/core/src/test/org/apache/solr/cli/ExportToolTest.java b/solr/core/src/test/org/apache/solr/cli/ExportToolTest.javanew file mode 100644index 00000000000..c5ceffa4cc0--- /dev/null+++ b/solr/core/src/test/org/apache/solr/cli/ExportToolTest.java@@ -0,0 +1,147 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.cli;++import java.nio.file.Files;+import java.nio.file.Path;+import java.util.List;+import org.apache.solr.client.solrj.request.AbstractUpdateRequest;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.request.UpdateRequest;+import org.apache.solr.cloud.SolrCloudTestCase;+import org.junit.BeforeClass;+import org.junit.Test;++public class ExportToolTest extends SolrCloudTestCase {+ private static final String COLLECTION = "exportToolColl";+ private static final int DOCS = 10;++ /** Runs the tool. Overridden by the picocli variant of this test. */+ protected int runTool(String[] args, Class<? extends ToolBase> clazz) throws Exception {+ return CLITestHelper.runTool(args, clazz);+ }++ @BeforeClass+ public static void setupCluster() throws Exception {+ configureCluster(1).addConfig("conf", configset("cloud-minimal")).configure();+ CollectionAdminRequest.createCollection(COLLECTION, "conf", 1, 1)+ .process(cluster.getSolrClient());+ cluster.waitForActiveCollection(COLLECTION, 1, 1);++ UpdateRequest ur = new UpdateRequest();+ ur.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);+ for (int i = 0; i < DOCS; i++) {+ ur.add("id", String.valueOf(i), "desc_s", "doc" + i);+ }+ ur.process(cluster.getSolrClient(), COLLECTION);+ }++ private String solrUrl() {+ return cluster.getJettySolrRunner(0).getBaseUrl().toString();+ }++ private List<String> exportLines(String... extraArgs) throws Exception {+ Path outDir = createTempDir();+ String[] fixed = {+ "export",+ "-c",+ COLLECTION,+ "--solr-url",+ solrUrl(),+ "--format",+ "jsonl",+ "--output",+ outDir.toString()+ };+ String[] args = new String[fixed.length + extraArgs.length];+ System.arraycopy(fixed, 0, args, 0, fixed.length);+ System.arraycopy(extraArgs, 0, args, fixed.length, extraArgs.length);++ assertEquals(0, runTool(args, ExportTool.class));+ return Files.readAllLines(outDir.resolve(COLLECTION + ".jsonl"));+ }++ @Test+ public void testExportAllDocs() throws Exception {+ assertEquals(DOCS, exportLines("--limit", "-1").size());+ }++ @Test+ public void testLimitDefaultsToOneHundredAndIsHonoured() throws Exception {+ assertEquals(DOCS, exportLines().size());+ assertEquals(3, exportLines("--limit", "3").size());+ }++ @Test+ public void testQueryAndFields() throws Exception {+ List<String> lines = exportLines("--query", "id:7", "--fields", "id");+ assertEquals(1, lines.size());+ assertTrue(lines.get(0), lines.get(0).contains("\"id\":\"7\""));+ assertFalse(lines.get(0), lines.get(0).contains("desc_s"));+ }++ @Test+ public void testFailsWithoutAConnectionTarget() throws Exception {+ // commons-cli reports 1; picocli reports its usage-error code 2+ assertNotEquals(0, runTool(new String[] {"export", "-c", COLLECTION}, ExportTool.class));+ }++ int exportTo(Path outDir, String... extraArgs) throws Exception {+ String[] fixed = {+ "export", "-c", COLLECTION, "--solr-url", solrUrl(), "--output", outDir.toString()+ };+ String[] args = new String[fixed.length + extraArgs.length];+ System.arraycopy(fixed, 0, args, 0, fixed.length);+ System.arraycopy(extraArgs, 0, args, fixed.length, extraArgs.length);+ return runTool(args, ExportTool.class);+ }++ @Test+ public void testFormatDefaultsToJson() throws Exception {+ Path outDir = createTempDir();+ assertEquals(0, exportTo(outDir));+ String json = Files.readString(outDir.resolve(COLLECTION + ".json"));+ assertTrue(json, json.contains("\"id\":\"1\""));+ }++ @Test+ public void testJavabinFormat() throws Exception {+ Path outDir = createTempDir();+ assertEquals(0, exportTo(outDir, "--format", "javabin"));+ assertTrue(Files.size(outDir.resolve(COLLECTION + ".javabin")) > 0);+ }++ @Test+ public void testUnknownFormatFails() throws Exception {+ Path outDir = createTempDir();+ // commons-cli reports 1; picocli reports its usage-error code 2+ assertNotEquals(0, exportTo(outDir, "--format", "xml"));+ try (var written = Files.list(outDir)) {+ assertEquals(List.of(), written.toList());+ }+ }++ @Test+ public void testNonNumericLimitFails() throws Exception {+ Path outDir = createTempDir();+ // commons-cli fails on parsing the number (1); picocli rejects it up front (2)+ assertNotEquals(0, exportTo(outDir, "--limit", "abc"));+ try (var written = Files.list(outDir)) {+ assertEquals(List.of(), written.toList());+ }+ }+}diff --git a/solr/solr-ref-guide/modules/deployment-guide/deployment-nav.adoc b/solr/solr-ref-guide/modules/deployment-guide/deployment-nav.adocindex 47f6a502c08..712f9fee593 100644--- a/solr/solr-ref-guide/modules/deployment-guide/deployment-nav.adoc+++ b/solr/solr-ref-guide/modules/deployment-guide/deployment-nav.adoc@@ -27,6 +27,7 @@*** xref:cli/solr-config.adoc[config]*** xref:cli/solr-create.adoc[create]*** xref:cli/solr-delete.adoc[delete]+*** xref:cli/solr-export.adoc[export]*** xref:cli/solr-start.adoc[start]*** xref:cli/solr-status.adoc[status]*** xref:cli/solr-stop.adoc[stop]diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/cli/index.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/cli/index.adocindex 600c2d842c5..051f9f6f1b7 100644--- a/solr/solr-ref-guide/modules/deployment-guide/pages/cli/index.adoc+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/cli/index.adoc@@ -66,6 +66,7 @@ Any command not listed here is unavailable when the picocli implementation is en|xref:cli/solr-config.adoc[bin/solr config]|xref:cli/solr-create.adoc[bin/solr create]|xref:cli/solr-delete.adoc[bin/solr delete]+|xref:cli/solr-export.adoc[bin/solr export]|xref:cli/solr-start.adoc[bin/solr start]|xref:cli/solr-status.adoc[bin/solr status]|xref:cli/solr-stop.adoc[bin/solr stop]diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/cli/solr-export.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/cli/solr-export.adocnew file mode 100644index 00000000000..74d126d3943--- /dev/null+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/cli/solr-export.adoc@@ -0,0 +1,101 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.+//+// DO NOT EDIT -- this page is auto-generated from picocli annotations.+// To update: modify the @Command/@Option annotations in the Java source, then run:+// ./gradlew :solr:solr-ref-guide:generateCliDocs++= bin/solr export+:page-toclevels: 2++[WARNING]+====+This page documents the *experimental* picocli implementation of `bin/solr`, which is not enabled by default.+For what `bin/solr` does by default, see xref:solr-control-script-reference.adoc[].+See xref:cli/index.adoc[] to enable this implementation and for its known limitations.+====++// tag::picocli-generated-man-section-synopsis[]+== Synopsis++....+bin/solr export [-hv] [--compress] -c=NAME [--fields=FIELDA,FIELDB]+ [--format=FORMAT] [--limit=#] [--output=PATH] [--query=QUERY]+ [-u=<credentials>] (-s=<solrConnection> | --solr-url=<solrUrl> |+ -z=<zkHost>)+....++// end::picocli-generated-man-section-synopsis[]++// tag::picocli-generated-man-section-description[]+== Description++Exports documents from a collection to a local file.++// end::picocli-generated-man-section-description[]++// tag::picocli-generated-man-section-options[]+== Options++*-c*, *--name*=_NAME_::+ *(required)* Name of the collection.++*--compress*::+ Compress the output. Defaults to false.++*--fields*=_FIELDA,FIELDB_::+ Comma separated list of fields to export. By default all fields are fetched.++*--format*=_FORMAT_::+ Output format for exported docs (json, jsonl, javabin), defaulting to json.++*-h*, *--help*::+ Print this help message and exit.++*--limit*=_#_::+ Maximum number of docs to download. Default is 100, use -1 for all docs.++*--output*=_PATH_::+ Path to output the exported data, and optionally the file name, defaults to 'collection-name'.++*--query*=_QUERY_::+ A custom query, default is '*:*'.++*-s*, *--solr-connection*=_<solrConnection>_::+ Zookeeper or HTTP(s) connection string; unnecessary if SOLR_CONNECTION is defined in solr.in.sh; otherwise, defaults to localhost:9983.++*--solr-url*=_<solrUrl>_::+ Base Solr URL, which can be used to determine the zk-host if that's not known.++*-u*, *--credentials*=_<credentials>_::+ Credentials in the format username:password. Example: --credentials solr:SolrRocks++*-v*, *--verbose*::+ Enable verbose mode.++*-z*, *--zk-host*=_<zkHost>_::+ Zookeeper connection string; unnecessary if ZK_HOST is defined in solr.in.sh; otherwise, defaults to localhost:9983.++// end::picocli-generated-man-section-options[]++// tag::picocli-generated-man-section-footer[]+== Examples+[%hardbreaks]+ # Export a collection's documents to gettingstarted.json+ bin/solr export -c gettingstarted --solr-url http://localhost:8983++// end::picocli-generated-man-section-footer[]