Apache Solr open-source search software
SOLR-17180: Deprecate snapshotscli.sh in favour of bin/solr sub commands. (#2381)
Eric Pugh5374d3428248
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 783da7f8d6f..4ccf6e5fdb5 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -108,6 +108,7 @@ New FeaturesImprovements---------------------* SOLR-17397: SkipExistingDocumentsProcessor now functions correctly with child documents. (Tim Owens via Eric Pugh)+* SOLR-17180: Deprecate snapshotscli.sh in favour of bin/solr snapshot sub commands. Now able to manage Snapshots from the CLI. HDFS module specific snapshot script now ships as part of that module in the modules/hdfs/bin directory. (Eric Pugh)Optimizations---------------------diff --git a/solr/core/src/java/org/apache/solr/cli/SnapshotCreateTool.java b/solr/core/src/java/org/apache/solr/cli/SnapshotCreateTool.javanew file mode 100644index 00000000000..41a5cf13a6b--- /dev/null+++ b/solr/core/src/java/org/apache/solr/cli/SnapshotCreateTool.java@@ -0,0 +1,103 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.cli;++import java.io.PrintStream;+import java.util.List;+import org.apache.commons.cli.CommandLine;+import org.apache.commons.cli.Option;+import org.apache.solr.client.solrj.SolrClient;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.response.CollectionAdminResponse;++/** Supports snapshot-create command in the bin/solr script. */+public class SnapshotCreateTool extends ToolBase {++ public SnapshotCreateTool() {+ this(CLIO.getOutStream());+ }++ public SnapshotCreateTool(PrintStream stdout) {+ super(stdout);+ }++ @Override+ public String getName() {+ return "snapshot-create";+ }++ @Override+ public List<Option> getOptions() {+ return List.of(+ SolrCLI.OPTION_ZKHOST,+ SolrCLI.OPTION_SOLRURL,+ Option.builder("c")+ .longOpt("name")+ .argName("NAME")+ .hasArg()+ .required(true)+ .desc("Name of collection to be snapshot.")+ .build(),+ Option.builder()+ .longOpt("snapshot-name")+ .argName("NAME")+ .hasArg()+ .required(true)+ .desc("Name of the snapshot to produce")+ .build(),+ SolrCLI.OPTION_CREDENTIALS,+ SolrCLI.OPTION_VERBOSE);+ }++ @Override+ public void runImpl(CommandLine cli) throws Exception {+ SolrCLI.raiseLogLevelUnlessVerbose(cli);++ String snapshotName = cli.getOptionValue("snapshot-name");+ String collectionName = cli.getOptionValue("name");+ try (var solrClient = SolrCLI.getSolrClient(cli)) {+ createSnapshot(solrClient, collectionName, snapshotName);+ }+ }++ public void createSnapshot(SolrClient solrClient, String collectionName, String snapshotName) {+ CollectionAdminRequest.CreateSnapshot createSnapshot =+ new CollectionAdminRequest.CreateSnapshot(collectionName, snapshotName);+ CollectionAdminResponse resp;+ try {+ resp = createSnapshot.process(solrClient);+ if (resp.getStatus() != 0) {+ throw new IllegalStateException(+ "The CREATESNAPSHOT request failed. The status code is " + resp.getStatus());+ }+ echo(+ "Successfully created snapshot with name "+ + snapshotName+ + " for collection "+ + collectionName);++ } catch (Exception e) {+ echo(+ "Failed to create a snapshot with name "+ + snapshotName+ + " for collection "+ + collectionName+ + " due to following error : "+ + e.getLocalizedMessage());+ }+ }+}diff --git a/solr/core/src/java/org/apache/solr/cli/SnapshotDeleteTool.java b/solr/core/src/java/org/apache/solr/cli/SnapshotDeleteTool.javanew file mode 100644index 00000000000..ee8a0506492--- /dev/null+++ b/solr/core/src/java/org/apache/solr/cli/SnapshotDeleteTool.java@@ -0,0 +1,103 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.cli;++import java.io.PrintStream;+import java.util.List;+import org.apache.commons.cli.CommandLine;+import org.apache.commons.cli.Option;+import org.apache.solr.client.solrj.SolrClient;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.response.CollectionAdminResponse;++/** Supports snapshot-delete command in the bin/solr script. */+public class SnapshotDeleteTool extends ToolBase {++ public SnapshotDeleteTool() {+ this(CLIO.getOutStream());+ }++ public SnapshotDeleteTool(PrintStream stdout) {+ super(stdout);+ }++ @Override+ public String getName() {+ return "snapshot-delete";+ }++ @Override+ public List<Option> getOptions() {+ return List.of(+ SolrCLI.OPTION_ZKHOST,+ SolrCLI.OPTION_SOLRURL,+ Option.builder("c")+ .longOpt("name")+ .argName("NAME")+ .hasArg()+ .required(true)+ .desc("Name of collection to manage.")+ .build(),+ Option.builder()+ .longOpt("snapshot-name")+ .argName("NAME")+ .hasArg()+ .required(true)+ .desc("Name of the snapshot to delete")+ .build(),+ SolrCLI.OPTION_CREDENTIALS,+ SolrCLI.OPTION_VERBOSE);+ }++ @Override+ public void runImpl(CommandLine cli) throws Exception {+ SolrCLI.raiseLogLevelUnlessVerbose(cli);++ String snapshotName = cli.getOptionValue("snapshot-name");+ String collectionName = cli.getOptionValue("name");+ try (var solrClient = SolrCLI.getSolrClient(cli)) {+ deleteSnapshot(solrClient, collectionName, snapshotName);+ }+ }++ public void deleteSnapshot(SolrClient solrClient, String collectionName, String snapshotName) {+ CollectionAdminRequest.DeleteSnapshot deleteSnapshot =+ new CollectionAdminRequest.DeleteSnapshot(collectionName, snapshotName);+ CollectionAdminResponse resp;+ try {+ resp = deleteSnapshot.process(solrClient);+ if (resp.getStatus() != 0) {+ throw new IllegalStateException(+ "The DELETESNAPSHOT request failed. The status code is " + resp.getStatus());+ }+ echo(+ "Successfully deleted snapshot with name "+ + snapshotName+ + " for collection "+ + collectionName);++ } catch (Exception e) {+ echo(+ "Failed to delete a snapshot with name "+ + snapshotName+ + " for collection "+ + collectionName+ + " due to following error : "+ + e.getLocalizedMessage());+ }+ }+}diff --git a/solr/core/src/java/org/apache/solr/cli/SnapshotDescribeTool.java b/solr/core/src/java/org/apache/solr/cli/SnapshotDescribeTool.javanew file mode 100644index 00000000000..94ef6d98a9e--- /dev/null+++ b/solr/core/src/java/org/apache/solr/cli/SnapshotDescribeTool.java@@ -0,0 +1,142 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.cli;++import java.io.IOException;+import java.io.PrintStream;+import java.text.DateFormat;+import java.text.SimpleDateFormat;+import java.util.ArrayList;+import java.util.Collection;+import java.util.List;+import java.util.Locale;+import org.apache.commons.cli.CommandLine;+import org.apache.commons.cli.Option;+import org.apache.solr.client.solrj.SolrClient;+import org.apache.solr.client.solrj.SolrServerException;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.response.CollectionAdminResponse;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.core.snapshots.CollectionSnapshotMetaData;+import org.apache.solr.core.snapshots.SolrSnapshotManager;++/** Supports snapshot-describe command in the bin/solr script. */+public class SnapshotDescribeTool extends ToolBase {++ public SnapshotDescribeTool() {+ this(CLIO.getOutStream());+ }++ public SnapshotDescribeTool(PrintStream stdout) {+ super(stdout);+ }++ private static final DateFormat dateFormat =+ new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z", Locale.getDefault());++ @Override+ public String getName() {+ return "snapshot-describe";+ }++ @Override+ public List<Option> getOptions() {+ return List.of(+ SolrCLI.OPTION_ZKHOST,+ SolrCLI.OPTION_SOLRURL,+ Option.builder("c")+ .longOpt("name")+ .argName("NAME")+ .hasArg()+ .required(true)+ .desc("Name of collection to be snapshot.")+ .build(),+ Option.builder()+ .longOpt("snapshot-name")+ .argName("NAME")+ .hasArg()+ .required(true)+ .desc("Name of the snapshot to describe")+ .build(),+ SolrCLI.OPTION_CREDENTIALS,+ SolrCLI.OPTION_VERBOSE);+ }++ @Override+ public void runImpl(CommandLine cli) throws Exception {+ SolrCLI.raiseLogLevelUnlessVerbose(cli);++ String snapshotName = cli.getOptionValue("snapshot-name");+ String collectionName = cli.getOptionValue("name");+ try (var solrClient = SolrCLI.getSolrClient(cli)) {+ describeSnapshot(solrClient, collectionName, snapshotName);+ }+ }++ public void describeSnapshot(SolrClient solrClient, String collectionName, String snapshotName) {+ try {+ Collection<CollectionSnapshotMetaData> snaps =+ listCollectionSnapshots(solrClient, collectionName);+ for (CollectionSnapshotMetaData m : snaps) {+ if (snapshotName.equals(m.getName())) {+ echo("Name: " + m.getName());+ echo("Status: " + m.getStatus());+ echo("Time of creation: " + dateFormat.format(m.getCreationDate()));+ echo("Total number of cores with snapshot: " + m.getReplicaSnapshots().size());+ echo("-----------------------------------");+ for (CollectionSnapshotMetaData.CoreSnapshotMetaData n : m.getReplicaSnapshots()) {+ String builder =+ "Core [name="+ + n.getCoreName()+ + ", leader="+ + n.isLeader()+ + ", generation="+ + n.getGenerationNumber()+ + ", indexDirPath="+ + n.getIndexDirPath()+ + "]\n";+ echo(builder);+ }+ }+ }+ } catch (Exception e) {+ echo("Failed to fetch snapshot details due to following error : " + e.getLocalizedMessage());+ }+ }++ private Collection<CollectionSnapshotMetaData> listCollectionSnapshots(+ SolrClient solrClient, String collectionName) throws SolrServerException, IOException {+ CollectionAdminRequest.ListSnapshots listSnapshots =+ new CollectionAdminRequest.ListSnapshots(collectionName);+ CollectionAdminResponse resp = listSnapshots.process(solrClient);++ if (resp.getStatus() != 0) {+ throw new IllegalStateException(+ "The LISTSNAPSHOTS request failed. The status code is " + resp.getStatus());+ }++ NamedList<?> apiResult =+ (NamedList<?>) resp.getResponse().get(SolrSnapshotManager.SNAPSHOTS_INFO);++ Collection<CollectionSnapshotMetaData> result = new ArrayList<>();+ for (int i = 0; i < apiResult.size(); i++) {+ result.add(new CollectionSnapshotMetaData((NamedList<?>) apiResult.getVal(i)));+ }++ return result;+ }+}diff --git a/solr/core/src/java/org/apache/solr/cli/SnapshotExportTool.java b/solr/core/src/java/org/apache/solr/cli/SnapshotExportTool.javanew file mode 100644index 00000000000..d38a1b19b69--- /dev/null+++ b/solr/core/src/java/org/apache/solr/cli/SnapshotExportTool.java@@ -0,0 +1,132 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.cli;++import java.io.PrintStream;+import java.util.List;+import java.util.Optional;+import org.apache.commons.cli.CommandLine;+import org.apache.commons.cli.Option;+import org.apache.solr.client.solrj.SolrClient;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.common.params.CollectionAdminParams;++/** Supports snapshot-export command in the bin/solr script. */+public class SnapshotExportTool extends ToolBase {++ public SnapshotExportTool() {+ this(CLIO.getOutStream());+ }++ public SnapshotExportTool(PrintStream stdout) {+ super(stdout);+ }++ @Override+ public String getName() {+ return "snapshot-export";+ }++ @Override+ public List<Option> getOptions() {+ return List.of(+ SolrCLI.OPTION_ZKHOST,+ SolrCLI.OPTION_SOLRURL,+ Option.builder("c")+ .longOpt("name")+ .argName("NAME")+ .hasArg()+ .required(true)+ .desc("Name of collection to be snapshot.")+ .build(),+ Option.builder()+ .longOpt("snapshot-name")+ .argName("NAME")+ .hasArg()+ .required(true)+ .desc("Name of the snapshot to be exported.")+ .build(),+ Option.builder()+ .longOpt("dest-dir")+ .argName("DIR")+ .hasArg()+ .required(true)+ .desc(+ "Path of a temporary directory on local filesystem during snapshot export command.")+ .build(),+ Option.builder()+ .longOpt("backup-repo-name")+ .argName("DIR")+ .hasArg()+ .required(false)+ .desc(+ "Specifies name of the backup repository to be used during snapshot export preparation.")+ .build(),+ Option.builder("i")+ .longOpt("async-id")+ .argName("ID")+ .hasArg()+ .required(false)+ .desc(+ "Specifies the async request identifier to be used during snapshot export preparation.")+ .build(),+ SolrCLI.OPTION_CREDENTIALS,+ SolrCLI.OPTION_VERBOSE);+ }++ @Override+ public void runImpl(CommandLine cli) throws Exception {+ SolrCLI.raiseLogLevelUnlessVerbose(cli);+ //+ String snapshotName = cli.getOptionValue("snapshot-name");+ String collectionName = cli.getOptionValue("name");+ String destDir = cli.getOptionValue("dest-dir");+ Optional<String> backupRepo = Optional.ofNullable(cli.getOptionValue("backup-repo-name"));+ Optional<String> asyncReqId = Optional.ofNullable(cli.getOptionValue("async-id"));++ try (var solrClient = SolrCLI.getSolrClient(cli)) {+ exportSnapshot(solrClient, collectionName, snapshotName, destDir, backupRepo, asyncReqId);+ }+ }++ public void exportSnapshot(+ SolrClient solrClient,+ String collectionName,+ String snapshotName,+ String destPath,+ Optional<String> backupRepo,+ Optional<String> asyncReqId) {+ try {+ CollectionAdminRequest.Backup backup =+ new CollectionAdminRequest.Backup(collectionName, snapshotName);+ backup.setCommitName(snapshotName);+ backup.setIndexBackupStrategy(CollectionAdminParams.COPY_FILES_STRATEGY);+ backup.setLocation(destPath);+ if (backupRepo.isPresent()) {+ backup.setRepositoryName(backupRepo.get());+ }+ // if asyncId is null, processAsync will block and throw an Exception with any error+ backup.processAsync(asyncReqId.orElse(null), solrClient);+ } catch (Exception e) {+ throw new IllegalStateException(+ "Failed to backup collection meta-data for collection "+ + collectionName+ + " due to following error : "+ + e.getLocalizedMessage());+ }+ }+}diff --git a/solr/core/src/java/org/apache/solr/cli/SnapshotListTool.java b/solr/core/src/java/org/apache/solr/cli/SnapshotListTool.javanew file mode 100644index 00000000000..04cd0738825--- /dev/null+++ b/solr/core/src/java/org/apache/solr/cli/SnapshotListTool.java@@ -0,0 +1,96 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.cli;++import java.io.PrintStream;+import java.util.List;+import org.apache.commons.cli.CommandLine;+import org.apache.commons.cli.Option;+import org.apache.solr.client.solrj.SolrClient;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.response.CollectionAdminResponse;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.core.snapshots.SolrSnapshotManager;++/** Supports snapshot-list command in the bin/solr script. */+public class SnapshotListTool extends ToolBase {++ public SnapshotListTool() {+ this(CLIO.getOutStream());+ }++ public SnapshotListTool(PrintStream stdout) {+ super(stdout);+ }++ @Override+ public String getName() {+ return "snapshot-list";+ }++ @Override+ public List<Option> getOptions() {+ return List.of(+ SolrCLI.OPTION_ZKHOST,+ SolrCLI.OPTION_SOLRURL,+ Option.builder("c")+ .longOpt("name")+ .argName("NAME")+ .hasArg()+ .required(true)+ .desc("Name of collection to list snapshots for.")+ .build(),+ SolrCLI.OPTION_CREDENTIALS,+ SolrCLI.OPTION_VERBOSE);+ }++ @Override+ public void runImpl(CommandLine cli) throws Exception {+ SolrCLI.raiseLogLevelUnlessVerbose(cli);++ String collectionName = cli.getOptionValue("name");+ try (var solrClient = SolrCLI.getSolrClient(cli)) {+ listSnapshots(solrClient, collectionName);+ }+ }++ public void listSnapshots(SolrClient solrClient, String collectionName) {+ CollectionAdminRequest.ListSnapshots listSnaps =+ new CollectionAdminRequest.ListSnapshots(collectionName);+ CollectionAdminResponse resp;+ try {+ resp = listSnaps.process(solrClient);+ if (resp.getStatus() != 0) {+ throw new IllegalStateException(+ "The LISTSNAPSHOTS request failed. The status code is " + resp.getStatus());+ }++ NamedList<?> apiResult =+ (NamedList<?>) resp.getResponse().get(SolrSnapshotManager.SNAPSHOTS_INFO);+ for (int i = 0; i < apiResult.size(); i++) {+ echo(apiResult.getName(i));+ }++ } catch (Exception e) {+ echo(+ "Failed to list snapshots for collection "+ + collectionName+ + " due to following error : "+ + e.getLocalizedMessage());+ }+ }+}diff --git a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java b/solr/core/src/java/org/apache/solr/cli/SolrCLI.javaindex b7a74bb21af..05488e7d8e3 100755--- a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java+++ b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java@@ -340,6 +340,11 @@ public class SolrCLI implements CLIO {else if ("post".equals(toolType)) return new PostTool();else if ("postlogs".equals(toolType)) return new PostLogsTool();else if ("version".equals(toolType)) return new VersionTool();+ else if ("snapshot-create".equals(toolType)) return new SnapshotCreateTool();+ else if ("snapshot-delete".equals(toolType)) return new SnapshotDeleteTool();+ else if ("snapshot-list".equals(toolType)) return new SnapshotListTool();+ else if ("snapshot-describe".equals(toolType)) return new SnapshotDescribeTool();+ else if ("snapshot-export".equals(toolType)) return new SnapshotExportTool();// If you add a built-in tool to this class, add it here to avoid// classpath scanning@@ -587,7 +592,11 @@ public class SolrCLI implements CLIO {print("Usage: solr COMMAND OPTIONS");print(- " where COMMAND is one of: start, stop, restart, status, healthcheck, create, delete, version, zk, auth, assert, config, export, api, package, post");+ " where COMMAND is one of: start, stop, restart, status, healthcheck, create, delete, version, auth, assert, config, export, api, package, post, ");+ print(+ " zk ls, zk cp, zk rm , zk mv, zk mkroot, zk upconfig, zk downconfig,");+ print(+ " snapshot-create, snapshot-list, snapshot-delete, snapshot-export, snapshot-prepare-export");print("");print(" Standalone server example (start Solr running in the background on port 8984):");print("");diff --git a/solr/modules/hdfs/bin/prepare-snapshot-export.sh b/solr/modules/hdfs/bin/prepare-snapshot-export.shnew file mode 100755index 00000000000..7de91e19197--- /dev/null+++ b/solr/modules/hdfs/bin/prepare-snapshot-export.sh@@ -0,0 +1,188 @@+#!/usr/bin/env bash+# Licensed to the Apache Software Foundation (ASF) under one or more+# contributor license agreements. See the NOTICE file distributed with+# this work for additional information regarding copyright ownership.+# The ASF licenses this file to You under the Apache License, Version 2.0+# (the "License"); you may not use this file except in compliance with+# the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++set -e++usage() {+ echo "This script is to support the specific use case of using Hadoop specific libraries for managing Solr snapshot."+ echo "--prepare-snapshot-export <arg> This command will prepare"+ echo " copylistings for the specified"+ echo " snapshot. This command should only"+ echo " be used only if Solr is deployed"+ echo " with Hadoop and collection index"+ echo " files are stored on a shared"+ echo " file-system e.g. HDFS"+ echo "--export <arg> This command will create a backup"+ echo " for the specified snapshot."+}++distcp_warning() {+ echo "SOLR_USE_DISTCP environment variable is not set. \+ Do you want to use hadoop distcp tool for exporting Solr collection snapshot ?"+}++use_bin_solr_warning() {+ echo "Please use the bin/solr script to execute this command."+}++parse_options() {+ OPTIND=3+ while getopts ":c:d:s:" o ; do+ case "${o}" in+ d)+ destPath=${OPTARG}+ ;;+ s)+ sourcePath=${OPTARG}+ ;;+ c)+ collectionName=${OPTARG}+ ;;+ *)+ echo "Unknown option ${OPTARG}"+ usage 1>&2+ exit 1+ ;;+ esac+ done+}++run_solr_snapshot_tool() {+ JVM="java"+ scriptDir=$(dirname "$0")+ if [ -n "$LOG4J_PROPS" ]; then+ log4j_config="file:${LOG4J_PROPS}"+ else+ log4j_config="file:${scriptDir}/../../../server/resources/log4j2-console.xml"+ fi+ PATH=${JAVA_HOME}/bin:${PATH} ${JVM} ${ZKCLI_JVM_FLAGS} -Dlog4j.configurationFile=${log4j_config} \+ -classpath "${solrLibPath}" org.apache.solr.hdfs.snapshots.SolrSnapshotsTool "$@" 2> /dev/null+}++prepare_snapshot_export() {+ #Make sure to cleanup the temporary files.+ scratch=$(mktemp -d -t solrsnaps.XXXXXXXXXX)+ function finish {+ rm -rf "${scratch}"+ }+ trap finish EXIT++ if hdfs dfs -test -d "${destPath}" ; then+ run_solr_snapshot_tool --prepare-snapshot-export "$@" -t "${scratch}"++ hdfs dfs -mkdir -p "${copyListingDirPath}" > /dev/null+ find "${scratch}" -type f -printf "%f\n" | while read shardId; do+ echo "Copying the copy-listing for $shardId"+ hdfs dfs -copyFromLocal "${scratch}/${shardId}" "${copyListingDirPath}" > /dev/null+ done+ else+ echo "Directory ${destPath} does not exist."+ exit 1+ fi+}++copy_snapshot_files() {+ copylisting_dir_path="$1"++ if hdfs dfs -test -d "${copylisting_dir_path}" ; then+ for shardId in $(hdfs dfs -stat "%n" "${copylisting_dir_path}/*"); do+ oPath="${destPath}/${snapshotName}/snapshot.${shardId}"+ echo "Copying the index files for ${shardId} to ${oPath}"+ ${distCpCmd} -f "${copylisting_dir_path}/${shardId}" "${oPath}" > /dev/null+ done+ else+ echo "Directory ${copylisting_dir_path} does not exist."+ exit 1+ fi+}++collectionName=""+destPath=""+sourcePath=""+cmd="$1"+snapshotName="$2"+copyListingDirPath=""+distCpCmd="${SOLR_DISTCP_CMD:-hadoop distcp}"+scriptDir=$(dirname "$0")+solrLibPath="${SOLR_LIB_PATH:-${scriptDir}/../../../server/solr-webapp/webapp/WEB-INF/lib/*:${scriptDir}/../../../server/lib/ext/*:${scriptDir}/../lib/*}"+case "${cmd}" in+ --create)+ use_bin_solr_warning+ ;;+ --delete)+ use_bin_solr_warning+ ;;+ --list)+ use_bin_solr_warning+ ;;+ --describe)+ use_bin_solr_warning+ ;;+ --prepare-snapshot-export)+ : "${SOLR_USE_DISTCP:? $(distcp_warning)}"++ parse_options "$@"++ : "${destPath:? Please specify destination directory using -d option}"++ copyListingDirPath="${destPath}/copylistings"+ prepare_snapshot_export "${@:2}"+ echo "Done. GoodBye!"+ ;;+ --export)+ if [ -z "${SOLR_USE_DISTCP}" ]; then+ echo "Not using the distcp command"+ use_bin_solr_warning+ echo "Done. GoodBye!"+ exit 0+ fi++ parse_options "$@"++ : "${snapshotName:? Please specify the name of the snapshot}"+ : "${destPath:? Please specify destination directory using -d option}"++ if [ -n "${collectionName}" ] && [ -n "${sourcePath}" ]; then+ echo "The -c and -s options can not be specified together"+ exit 1+ fi++ if [ -z "${collectionName}" ] && [ -z "${sourcePath}" ]; then+ echo "At least one of options (-c or -s) must be specified"+ exit 1+ fi++ if [ -n "${collectionName}" ]; then+ copyListingDirPath="${destPath}/${snapshotName}/copylistings"+ prepare_snapshot_export "${@:2}"+ copy_snapshot_files "${destPath}/${snapshotName}/copylistings"+ hdfs dfs -rm -r -f -skipTrash "${destPath}/${snapshotName}/copylistings" > /dev/null+ else+ copy_snapshot_files "${sourcePath}/copylistings"+ echo "Copying the collection meta-data to ${destPath}/${snapshotName}"+ ${distCpCmd} "${sourcePath}/${snapshotName}/*" "${destPath}/${snapshotName}/" > /dev/null+ fi++ echo "Done. GoodBye!"+ ;;+ --help|-h)+ usage 1>&2+ ;;+ *)+ echo "Unknown command ${cmd}"+ usage 1>&2+ exit 1+esacdiff --git a/solr/modules/hdfs/build.gradle b/solr/modules/hdfs/build.gradleindex 6ce2ee9d47f..c9cd4576e92 100644--- a/solr/modules/hdfs/build.gradle+++ b/solr/modules/hdfs/build.gradle@@ -94,3 +94,10 @@ tasks.withType(Test).configureEach {jvmArgs(["--add-opens=java.base/java.lang=ALL-UNNAMED","--add-opens=java.base/java.util=ALL-UNNAMED"])}++assemblePackaging {+ // Add bin folder to default packaging.+ from(projectDir, {+ include "bin/**"+ })+}diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/SolrSnapshotsTool.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/SolrSnapshotsTool.javaindex 5f903cbd932..ee49025061c 100644--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/SolrSnapshotsTool.java+++ b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/SolrSnapshotsTool.java@@ -112,6 +112,11 @@ public class SolrSnapshotsTool implements Closeable, CLIO {}}+ /**+ * @deprecated use equivalent snapshot related tool in {@link+ * org.apache.solr.cli.SnapshotCreateTool}+ */+ @Deprecated(since = "9.8.0")public void createSnapshot(String collectionName, String snapshotName) {CollectionAdminRequest.CreateSnapshot createSnap =new CollectionAdminRequest.CreateSnapshot(collectionName, snapshotName);@@ -144,6 +149,11 @@ public class SolrSnapshotsTool implements Closeable, CLIO {}}+ /**+ * @deprecated use equivalent snapshot related tool in {@link+ * org.apache.solr.cli.SnapshotDeleteTool}+ */+ @Deprecated(since = "9.8.0")public void deleteSnapshot(String collectionName, String snapshotName) {CollectionAdminRequest.DeleteSnapshot deleteSnap =new CollectionAdminRequest.DeleteSnapshot(collectionName, snapshotName);@@ -180,6 +190,11 @@ public class SolrSnapshotsTool implements Closeable, CLIO {}}+ /**+ * @deprecated use equivalent snapshot related tool in {@link+ * org.apache.solr.cli.SnapshotListTool}+ */+ @Deprecated(since = "9.8.0")public void listSnapshots(String collectionName) {CollectionAdminRequest.ListSnapshots listSnaps =new CollectionAdminRequest.ListSnapshots(collectionName);@@ -204,6 +219,11 @@ public class SolrSnapshotsTool implements Closeable, CLIO {}}+ /**+ * @deprecated use equivalent snapshot related tool in {@link+ * org.apache.solr.cli.SnapshotDescribeTool}+ */+ @Deprecated(since = "9.8.0")public void describeSnapshot(String collectionName, String snapshotName) {try {Collection<CollectionSnapshotMetaData> snaps = listCollectionSnapshots(collectionName);@@ -332,6 +352,8 @@ public class SolrSnapshotsTool implements Closeable, CLIO {}/**+ * HDFS specific step required before exporting a snapshot+ ** @param pathPrefix optional*/public void prepareForExport(@@ -375,6 +397,11 @@ public class SolrSnapshotsTool implements Closeable, CLIO {}}+ /**+ * @deprecated use equivalent snapshot related tool in {@link+ * org.apache.solr.cli.SnapshotExportTool}+ */+ @Deprecated(since = "9.8.0")public void exportSnapshot(String collectionName,String snapshotName,diff --git a/solr/packaging/test/test_snapshots.bats b/solr/packaging/test/test_snapshots.batsnew file mode 100644index 00000000000..4a6135bbd01--- /dev/null+++ b/solr/packaging/test/test_snapshots.bats@@ -0,0 +1,77 @@+#!/usr/bin/env bats++# Licensed to the Apache Software Foundation (ASF) under one or more+# contributor license agreements. See the NOTICE file distributed with+# this work for additional information regarding copyright ownership.+# The ASF licenses this file to You under the Apache License, Version 2.0+# (the "License"); you may not use this file except in compliance with+# the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++load bats_helper++setup_file() {+ common_clean_setup++ solr start -c -e films++}++teardown_file() {+ common_setup+ solr stop -all++}++setup() {+ common_setup+}++teardown() {+ # save a snapshot of SOLR_HOME for failed tests+ save_home_on_failure++}++@test "snapshot lifecycle" {+ # the way snapshots live, if you run create twice you get an error because snapshot already exists+ run solr snapshot-create -c films --snapshot-name snapshot1 --solr-url http://localhost:${SOLR_PORT}+ assert_output --partial "Successfully created snapshot with name snapshot1 for collection films"++ run solr snapshot-delete -c films --snapshot-name snapshot1 -url http://localhost:${SOLR_PORT}+ assert_output --partial "Successfully deleted snapshot with name snapshot1 for collection films"++ # make sure you can create it again!+ run solr snapshot-create -c films --snapshot-name snapshot1 -z localhost:${ZK_PORT}+ assert_output --partial "Successfully created snapshot with name snapshot1 for collection films"++ run solr snapshot-delete -c films --snapshot-name snapshot1+ assert_output --partial "Successfully deleted snapshot with name snapshot1 for collection films"+}++@test "snapshot list" {+ solr snapshot-create -c films --snapshot-name snapshot3 --solr-url http://localhost:${SOLR_PORT}++ run solr snapshot-list -c films -url http://localhost:${SOLR_PORT}/solr+ assert_output --partial "snapshot3"++ run solr snapshot-delete -c films --snapshot-name snapshot3 -url http://localhost:${SOLR_PORT}+ assert_output --partial "Successfully deleted snapshot with name snapshot3 for collection films"+}++@test "snapshot describe" {+ solr snapshot-create -c films --snapshot-name snapshot4 -url http://localhost:${SOLR_PORT}++ run solr snapshot-describe -c films --snapshot-name snapshot4+ assert_output --partial "Name: snapshot4"++ run solr snapshot-delete -c films --snapshot-name snapshot4+ assert_output --partial "Successfully deleted snapshot with name snapshot4 for collection films"+}diff --git a/solr/server/README.md b/solr/server/README.mdindex dd93b004706..45267a9ede3 100644--- a/solr/server/README.md+++ b/solr/server/README.md@@ -54,10 +54,6 @@ server/resourcesContains configuration files, such as the Log4j configuration (log4j2.xml) for configuring Solr loggers.-server/scripts/cloud-scripts-- Command-line utility for working with snapshots using snapshotscli.sh-server/solrDefault solr.solr.home directory where Solr will create core directories; must contain solr.xmldiff --git a/solr/server/scripts/cloud-scripts/snapshotscli.sh b/solr/server/scripts/cloud-scripts/snapshotscli.shindex 16c59837acd..b18274fbe2e 100755--- a/solr/server/scripts/cloud-scripts/snapshotscli.sh+++ b/solr/server/scripts/cloud-scripts/snapshotscli.sh@@ -2,6 +2,8 @@set -e+echo "This command is deprecated in favour of bin/solr equivalents and will be removed in Solr 10."+run_solr_snapshot_tool() {JVM="java"scriptDir=$(dirname "$0")@@ -173,4 +175,3 @@ case "${cmd}" inusage 1>&2exit 1esac-diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adocindex 4f15f8de677..29706fcfa74 100644--- a/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc@@ -1834,3 +1834,69 @@ Here is an example of sending a SQL query to the techproducts `/sql` end point ($ bin/solr api --solr-url http://localhost:8983/solr/techproducts/sql?stmt=select+id+from+techproducts+limit+10Results are streamed to the terminal.+++== Managing Solr Packages++The `package` command allows you to interact with Solr's xref:configuration-guide:package-manager.adoc[Package Manager].+++== Snapshots and Backups++The snapshots capablity of the CLI allows you to:++* Create snapshotted view of your index+* List all the available snapshots+* Provide details about individual Snapshots+* Export snapshots as part of a backup strategy+* Delete existing snapshots++Learn more about on the xref:backup-restore.adoc[Backup and Restore] and xref:collection-management.adoc[Collection Management] pages.++=== Create Snapshotted View++To create a snapshot you just give it the collection and the name you want:++[,console]+----+$ bin/solr snapshot-create -c <collection-name> --snapshot-name <snapshot-name>+----++=== List all available snapshots++To list all the snapshots that have been taken for a collection:++[,console]+----+$ bin/solr snapshot-list -c <collection-name>+----++=== Detailed information about a Snapshot++Use the describe command to gain detailed information about a specific snapshot:++[,console]+----+$ bin/solr snapshot-describe -c <collection-name> --snapshot-name <snapshot-name>+----++=== Converting a Snapshot to a Backup++Use the export command to take :++[,console]+----+$ bin/solr snapshot-export [--backup-repo-name <DIR>] -c <NAME> --dest-dir <DIR> [-i <ID>] --snapshot-name <NAME>+----++The `-i` parameter specifies that this is an asynchronous process.+++=== Delete a Snapshot++Use the delete command to delete a snapshot :++[,console]+----+$ bin/solr snapshot-delete -c <NAME> --snapshot-name <NAME>+----