Apache Solr open-source search software
SOLR-17400: remove deprecated snapshot related code (#2659)
Eric Pugh46f1bdf208e1
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 4ccf6e5fdb5..d369b73c87b 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -69,6 +69,8 @@ Deprecation Removals* SOLR-17313: Remove deprecated class SolrLogPostTool. PostLogsTool and the bin/solr post command has replaced this. (Eric Pugh)+* SOLR-17400: Remove deprecated script snapshotcli.sh. bin/solr snapshot-* commands have replaced this. (Eric Pugh)+Dependency Upgrades---------------------(No changes)diff --git a/solr/modules/hdfs/bin/prepare-snapshot-export.sh b/solr/modules/hdfs/bin/prepare-snapshot-export.shindex 7de91e19197..c74aaf7c627 100755--- a/solr/modules/hdfs/bin/prepare-snapshot-export.sh+++ b/solr/modules/hdfs/bin/prepare-snapshot-export.sh@@ -25,8 +25,6 @@ usage() {echo " with Hadoop and collection index"echo " files are stored on a shared"echo " file-system e.g. HDFS"- echo "--export <arg> This command will create a backup"- echo " for the specified snapshot."}distcp_warning() {@@ -69,7 +67,7 @@ run_solr_snapshot_tool() {log4j_config="file:${scriptDir}/../../../server/resources/log4j2-console.xml"fiPATH=${JAVA_HOME}/bin:${PATH} ${JVM} ${ZKCLI_JVM_FLAGS} -Dlog4j.configurationFile=${log4j_config} \- -classpath "${solrLibPath}" org.apache.solr.hdfs.snapshots.SolrSnapshotsTool "$@" 2> /dev/null+ -classpath "${solrLibPath}" org.apache.solr.hdfs.snapshots.SolrOnHdfsSnapshotsTool "$@" 2> /dev/null}prepare_snapshot_export() {diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/SolrSnapshotsTool.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/SolrOnHdfsSnapshotsTool.javasimilarity index 55%rename from solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/SolrSnapshotsTool.javarename to solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/SolrOnHdfsSnapshotsTool.javaindex ee49025061c..c0ce936da1d 100644--- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/SolrSnapshotsTool.java+++ b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/SolrOnHdfsSnapshotsTool.java@@ -24,8 +24,6 @@ import java.net.URI;import java.net.URISyntaxException;import java.nio.file.Files;import java.nio.file.Paths;-import java.text.DateFormat;-import java.text.SimpleDateFormat;import java.util.ArrayList;import java.util.Arrays;import java.util.Collection;@@ -33,7 +31,6 @@ import java.util.Collections;import java.util.Comparator;import java.util.HashMap;import java.util.List;-import java.util.Locale;import java.util.Map;import java.util.Optional;import org.apache.commons.cli.CommandLine;@@ -60,46 +57,36 @@ import org.apache.solr.core.snapshots.SolrSnapshotManager;import org.slf4j.Logger;import org.slf4j.LoggerFactory;-/** This class provides utility functions required for Solr snapshots functionality. */-public class SolrSnapshotsTool implements Closeable, CLIO {+/**+ * This class provides utility functions required for Solr on HDFS specific snapshots'+ * functionality.+ *+ * <p>For general purpose snapshot tooling see the related classes in the {@link+ * org.apache.solr.cli} package.+ */+public class SolrOnHdfsSnapshotsTool implements Closeable, CLIO {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());- private static final DateFormat dateFormat =- new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z", Locale.getDefault());- private static final String CREATE = "create";- private static final String DELETE = "delete";- private static final String LIST = "list";- private static final String DESCRIBE = "describe";private static final String PREPARE_FOR_EXPORT = "prepare-snapshot-export";- private static final String EXPORT_SNAPSHOT = "export";private static final String HELP = "help";private static final String COLLECTION = "c";private static final String TEMP_DIR = "t";private static final String DEST_DIR = "d";private static final String SOLR_ZK_ENSEMBLE = "z";private static final String HDFS_PATH_PREFIX = "p";- private static final String BACKUP_REPO_NAME = "r";- private static final String ASYNC_REQ_ID = "i";private static final List<String> OPTION_HELP_ORDER =Arrays.asList(- CREATE,- DELETE,- LIST,- DESCRIBE,PREPARE_FOR_EXPORT,- EXPORT_SNAPSHOT,HELP,SOLR_ZK_ENSEMBLE,COLLECTION,DEST_DIR,- BACKUP_REPO_NAME,- ASYNC_REQ_ID,TEMP_DIR,HDFS_PATH_PREFIX);private final CloudSolrClient solrClient;- public SolrSnapshotsTool(String solrZkEnsemble) {+ public SolrOnHdfsSnapshotsTool(String solrZkEnsemble) {solrClient =new CloudSolrClient.Builder(Collections.singletonList(solrZkEnsemble), Optional.empty()).build();@@ -112,77 +99,6 @@ public class SolrSnapshotsTool implements Closeable, CLIO {}}- /**- * @deprecated use equivalent snapshot related tool in {@link- * org.apache.solr.cli.SnapshotCreateTool}- */- @Deprecated(since = "9.8.0")- public void createSnapshot(String collectionName, String snapshotName) {- CollectionAdminRequest.CreateSnapshot createSnap =- new CollectionAdminRequest.CreateSnapshot(collectionName, snapshotName);- CollectionAdminResponse resp;- try {- resp = createSnap.process(solrClient);- if (resp.getStatus() != 0) {- throw new IllegalStateException(- "The CREATESNAPSHOT request failed. The status code is " + resp.getStatus());- }- CLIO.out(- "Successfully created snapshot with name "- + snapshotName- + " for collection "- + collectionName);-- } catch (Exception e) {- log.error(- "Failed to create a snapshot with name {} for collection {}",- snapshotName,- collectionName,- e);- CLIO.out(- "Failed to create a snapshot with name "- + snapshotName- + " for collection "- + collectionName- + " due to following error : "- + e.getLocalizedMessage());- }- }-- /**- * @deprecated use equivalent snapshot related tool in {@link- * org.apache.solr.cli.SnapshotDeleteTool}- */- @Deprecated(since = "9.8.0")- public void deleteSnapshot(String collectionName, String snapshotName) {- CollectionAdminRequest.DeleteSnapshot deleteSnap =- new CollectionAdminRequest.DeleteSnapshot(collectionName, snapshotName);- CollectionAdminResponse resp;- try {- resp = deleteSnap.process(solrClient);- checkResponse(resp, "DELETESNAPSHOT");- CLIO.out(- "Successfully deleted snapshot with name "- + snapshotName- + " for collection "- + collectionName);-- } catch (Exception e) {- log.error(- "Failed to delete a snapshot with name {} for collection {}",- snapshotName,- collectionName,- e);- CLIO.out(- "Failed to delete a snapshot with name "- + snapshotName- + " for collection "- + collectionName- + " due to following error : "- + e.getLocalizedMessage());- }- }-private static void checkResponse(CollectionAdminResponse resp, String requestType) {if (resp.getStatus() != 0) {throw new IllegalStateException(@@ -190,72 +106,6 @@ public class SolrSnapshotsTool implements Closeable, CLIO {}}- /**- * @deprecated use equivalent snapshot related tool in {@link- * org.apache.solr.cli.SnapshotListTool}- */- @Deprecated(since = "9.8.0")- public void listSnapshots(String collectionName) {- CollectionAdminRequest.ListSnapshots listSnaps =- new CollectionAdminRequest.ListSnapshots(collectionName);- CollectionAdminResponse resp;- try {- resp = listSnaps.process(solrClient);- checkResponse(resp, "LISTSNAPSHOTS");-- NamedList<?> apiResult =- (NamedList<?>) resp.getResponse().get(SolrSnapshotManager.SNAPSHOTS_INFO);- for (int i = 0; i < apiResult.size(); i++) {- CLIO.out(apiResult.getName(i));- }-- } catch (Exception e) {- log.error("Failed to list snapshots for collection {}", collectionName, e);- CLIO.out(- "Failed to list snapshots for collection "- + collectionName- + " due to following error : "- + e.getLocalizedMessage());- }- }-- /**- * @deprecated use equivalent snapshot related tool in {@link- * org.apache.solr.cli.SnapshotDescribeTool}- */- @Deprecated(since = "9.8.0")- public void describeSnapshot(String collectionName, String snapshotName) {- try {- Collection<CollectionSnapshotMetaData> snaps = listCollectionSnapshots(collectionName);- for (CollectionSnapshotMetaData m : snaps) {- if (snapshotName.equals(m.getName())) {- CLIO.out("Name: " + m.getName());- CLIO.out("Status: " + m.getStatus());- CLIO.out("Time of creation: " + dateFormat.format(m.getCreationDate()));- CLIO.out("Total number of cores with snapshot: " + m.getReplicaSnapshots().size());- CLIO.out("-----------------------------------");- for (CoreSnapshotMetaData n : m.getReplicaSnapshots()) {- StringBuilder builder = new StringBuilder();- builder.append("Core [name=");- builder.append(n.getCoreName());- builder.append(", leader=");- builder.append(n.isLeader());- builder.append(", generation=");- builder.append(n.getGenerationNumber());- builder.append(", indexDirPath=");- builder.append(n.getIndexDirPath());- builder.append("]\n");- CLIO.out(builder.toString());- }- }- }- } catch (Exception e) {- log.error("Failed to fetch snapshot details", e);- CLIO.out(- "Failed to fetch snapshot details due to following error : " + e.getLocalizedMessage());- }- }-/*** @param pathPrefix optional*/@@ -397,57 +247,10 @@ public class SolrSnapshotsTool implements Closeable, CLIO {}}- /**- * @deprecated use equivalent snapshot related tool in {@link- * org.apache.solr.cli.SnapshotExportTool}- */- @Deprecated(since = "9.8.0")- public void exportSnapshot(- String collectionName,- String snapshotName,- String destPath,- Optional<String> backupRepo,- Optional<String> asyncReqId) {- try {- CollectionAdminRequest.Backup backup =- new CollectionAdminRequest.Backup(collectionName, snapshotName);- backup.setCommitName(snapshotName);- backup.setIndexBackupStrategy(CollectionAdminParams.COPY_FILES_STRATEGY);- backup.setLocation(destPath);- if (backupRepo.isPresent()) {- backup.setRepositoryName(backupRepo.get());- }- // if asyncId is null, processAsync will block and throw an Exception with any error- backup.processAsync(asyncReqId.orElse(null), solrClient);- } catch (Exception e) {- log.error("Failed to backup collection meta-data for collection {}", collectionName, e);- CLIO.out(- "Failed to backup collection meta-data for collection "- + collectionName- + " due to following error : "- + e.getLocalizedMessage());- System.exit(1);- }- }-public static void main(String[] args) throws IOException {CommandLineParser parser = new PosixParser();Options options = new Options();- options.addOption(- null, CREATE, true, "This command will create a snapshot with the specified name");- options.addOption(- null, DELETE, true, "This command will delete a snapshot with the specified name");- options.addOption(- null,- LIST,- false,- "This command will list all the named snapshots for the specified collection.");- options.addOption(- null,- DESCRIBE,- true,- "This command will print details for a named snapshot for the specified collection.");options.addOption(null,PREPARE_FOR_EXPORT,@@ -455,11 +258,7 @@ public class SolrSnapshotsTool implements Closeable, CLIO {"This command will prepare copylistings for the specified snapshot."+ " This command should only be used only if Solr is deployed with Hadoop and collection index files are stored on a shared"+ " file-system e.g. HDFS");- options.addOption(- null,- EXPORT_SNAPSHOT,- true,- "This command will create a backup for the specified snapshot.");+options.addOption(null,HELP,@@ -486,16 +285,6 @@ public class SolrSnapshotsTool implements Closeable, CLIO {true,"This parameter specifies the HDFS URI prefix to be used"+ " during snapshot export preparation. This is applicable only if the Solr collection index files are stored on HDFS.");- options.addOption(- BACKUP_REPO_NAME,- true,- "This parameter specifies the name of the backup repository to be used"- + " during snapshot export preparation");- options.addOption(- ASYNC_REQ_ID,- true,- "This parameter specifies the async request identifier to be used"- + " during snapshot export preparation");CommandLine cmd = null;try {@@ -506,34 +295,10 @@ public class SolrSnapshotsTool implements Closeable, CLIO {System.exit(1);}- if (cmd.hasOption(CREATE)- || cmd.hasOption(DELETE)- || cmd.hasOption(LIST)- || cmd.hasOption(DESCRIBE)- || cmd.hasOption(PREPARE_FOR_EXPORT)- || cmd.hasOption(EXPORT_SNAPSHOT)) {- try (SolrSnapshotsTool tool =- new SolrSnapshotsTool(requiredArg(options, cmd, SOLR_ZK_ENSEMBLE))) {- if (cmd.hasOption(CREATE)) {- String snapshotName = cmd.getOptionValue(CREATE);- String collectionName = requiredArg(options, cmd, COLLECTION);- tool.createSnapshot(collectionName, snapshotName);-- } else if (cmd.hasOption(DELETE)) {- String snapshotName = cmd.getOptionValue(DELETE);- String collectionName = requiredArg(options, cmd, COLLECTION);- tool.deleteSnapshot(collectionName, snapshotName);-- } else if (cmd.hasOption(LIST)) {- String collectionName = requiredArg(options, cmd, COLLECTION);- tool.listSnapshots(collectionName);-- } else if (cmd.hasOption(DESCRIBE)) {- String snapshotName = cmd.getOptionValue(DESCRIBE);- String collectionName = requiredArg(options, cmd, COLLECTION);- tool.describeSnapshot(collectionName, snapshotName);-- } else if (cmd.hasOption(PREPARE_FOR_EXPORT)) {+ if (cmd.hasOption(PREPARE_FOR_EXPORT)) {+ try (SolrOnHdfsSnapshotsTool tool =+ new SolrOnHdfsSnapshotsTool(requiredArg(options, cmd, SOLR_ZK_ENSEMBLE))) {+ if (cmd.hasOption(PREPARE_FOR_EXPORT)) {String snapshotName = cmd.getOptionValue(PREPARE_FOR_EXPORT);String collectionName = requiredArg(options, cmd, COLLECTION);String localFsDir = requiredArg(options, cmd, TEMP_DIR);@@ -553,15 +318,6 @@ public class SolrSnapshotsTool implements Closeable, CLIO {}}tool.prepareForExport(collectionName, snapshotName, localFsDir, pathPrefix, hdfsOpDir);-- } else if (cmd.hasOption(EXPORT_SNAPSHOT)) {- String snapshotName = cmd.getOptionValue(EXPORT_SNAPSHOT);- String collectionName = requiredArg(options, cmd, COLLECTION);- String destDir = requiredArg(options, cmd, DEST_DIR);- Optional<String> backupRepo = Optional.ofNullable(cmd.getOptionValue(BACKUP_REPO_NAME));- Optional<String> asyncReqId = Optional.ofNullable(cmd.getOptionValue(ASYNC_REQ_ID));-- tool.exportSnapshot(collectionName, snapshotName, destDir, backupRepo, asyncReqId);}}} else if (cmd.hasOption(HELP)) {@@ -610,21 +366,16 @@ public class SolrSnapshotsTool implements Closeable, CLIO {}private static void printHelp(Options options) {- StringBuilder helpFooter = new StringBuilder();- helpFooter.append("Examples: \n");- helpFooter.append("snapshotscli.sh --create snapshot-1 -c books -z localhost:2181 \n");- helpFooter.append("snapshotscli.sh --list -c books -z localhost:2181 \n");- helpFooter.append("snapshotscli.sh --describe snapshot-1 -c books -z localhost:2181 \n");- helpFooter.append(- "snapshotscli.sh --export snapshot-1 -c books -z localhost:2181 -b repo -l backupPath -i req_0 \n");- helpFooter.append("snapshotscli.sh --delete snapshot-1 -c books -z localhost:2181 \n");+ String helpFooter =+ "Examples: \n"+ + "prepare-snapshot-export.sh --prepare-snapshot-export snapshot-1 -c books -z localhost:2181 -b repo -l backupPath \n";HelpFormatter formatter = new HelpFormatter();- formatter.setOptionComparator(new OptionComarator<>());- formatter.printHelp("SolrSnapshotsTool", null, options, helpFooter.toString(), false);+ formatter.setOptionComparator(new OptionComparator<>());+ formatter.printHelp("SolrOnHdfsSnapshotsTool", null, options, helpFooter, false);}- private static class OptionComarator<T extends Option> implements Comparator<T> {+ private static class OptionComparator<T extends Option> implements Comparator<T> {@Overridepublic int compare(T o1, T o2) {diff --git a/solr/server/scripts/cloud-scripts/snapshotscli.sh b/solr/server/scripts/cloud-scripts/snapshotscli.shdeleted file mode 100755index b18274fbe2e..00000000000--- a/solr/server/scripts/cloud-scripts/snapshotscli.sh+++ /dev/null@@ -1,177 +0,0 @@-#!/usr/bin/env bash--set -e--echo "This command is deprecated in favour of bin/solr equivalents and will be removed in Solr 10."--run_solr_snapshot_tool() {- JVM="java"- scriptDir=$(dirname "$0")- if [ -n "$LOG4J_PROPS" ]; then- log4j_config="file:${LOG4J_PROPS}"- else- log4j_config="file:${scriptDir}/../../resources/log4j2-console.xml"- fi- PATH=${JAVA_HOME}/bin:${PATH} ${JVM} ${ZKCLI_JVM_FLAGS} -Dlog4j.configurationFile=${log4j_config} \- -classpath "${solrLibPath}" org.apache.solr.hdfs.snapshots.SolrSnapshotsTool "$@" 2> /dev/null-}--usage() {- run_solr_snapshot_tool --help-}--distcp_warning() {- echo "SOLR_USE_DISTCP environment variable is not set. \- Do you want to use hadoop distcp tool for exporting Solr collection snapshot ?"-}--parse_options() {- OPTIND=3- while getopts ":c:d:s:z:p:r:i:" o ; do- case "${o}" in- d)- destPath=${OPTARG}- ;;- s)- sourcePath=${OPTARG}- ;;- c)- collectionName=${OPTARG}- ;;- z)- solrZkEnsemble=${OPTARG}- ;;- p)- pathPrefix=${OPTARG}- ;;- r)- backupRepoName=${OPTARG}- ;;- i)- aysncReqId=${OPTARG}- ;;- *)- echo "Unknown option ${OPTARG}"- usage 1>&2- exit 1- ;;- esac- done-}--prepare_snapshot_export() {- #Make sure to cleanup the temporary files.- scratch=$(mktemp -d -t solrsnaps.XXXXXXXXXX)- function finish {- rm -rf "${scratch}"- }- trap finish EXIT-- if hdfs dfs -test -d "${destPath}" ; then- run_solr_snapshot_tool --prepare-snapshot-export "$@" -t "${scratch}"-- hdfs dfs -mkdir -p "${copyListingDirPath}" > /dev/null- find "${scratch}" -type f -printf "%f\n" | while read shardId; do- echo "Copying the copy-listing for $shardId"- hdfs dfs -copyFromLocal "${scratch}/${shardId}" "${copyListingDirPath}" > /dev/null- done- else- echo "Directory ${destPath} does not exist."- exit 1- fi-}--copy_snapshot_files() {- copylisting_dir_path="$1"-- if hdfs dfs -test -d "${copylisting_dir_path}" ; then- for shardId in $(hdfs dfs -stat "%n" "${copylisting_dir_path}/*"); do- oPath="${destPath}/${snapshotName}/snapshot.${shardId}"- echo "Copying the index files for ${shardId} to ${oPath}"- ${distCpCmd} -f "${copylisting_dir_path}/${shardId}" "${oPath}" > /dev/null- done- else- echo "Directory ${copylisting_dir_path} does not exist."- exit 1- fi-}--collectionName=""-solrZkEnsemble=""-pathPrefix=""-destPath=""-sourcePath=""-cmd="$1"-snapshotName="$2"-copyListingDirPath=""-distCpCmd="${SOLR_DISTCP_CMD:-hadoop distcp}"-scriptDir=$(dirname "$0")-solrLibPath="${SOLR_LIB_PATH:-${scriptDir}/../../solr-webapp/webapp/WEB-INF/lib/*:${scriptDir}/../../lib/ext/*:${scriptDir}/../../../modules/hdfs/lib/*}"--case "${cmd}" in- --create)- run_solr_snapshot_tool "$@"- ;;- --delete)- run_solr_snapshot_tool "$@"- ;;- --list)- run_solr_snapshot_tool "$@"- ;;- --describe)- run_solr_snapshot_tool "$@"- ;;- --prepare-snapshot-export)- : "${SOLR_USE_DISTCP:? $(distcp_warning)}"-- parse_options "$@"-- : "${destPath:? Please specify destination directory using -d option}"-- copyListingDirPath="${destPath}/copylistings"- prepare_snapshot_export "${@:2}"- echo "Done. GoodBye!"- ;;- --export)- if [ -z "${SOLR_USE_DISTCP}" ]; then- run_solr_snapshot_tool "$@"- echo "Done. GoodBye!"- exit 0- fi-- parse_options "$@"-- : "${snapshotName:? Please specify the name of the snapshot}"- : "${destPath:? Please specify destination directory using -d option}"-- if [ -n "${collectionName}" ] && [ -n "${sourcePath}" ]; then- echo "The -c and -s options can not be specified together"- exit 1- fi-- if [ -z "${collectionName}" ] && [ -z "${sourcePath}" ]; then- echo "At least one of options (-c or -s) must be specified"- exit 1- fi-- if [ -n "${collectionName}" ]; then- copyListingDirPath="${destPath}/${snapshotName}/copylistings"- prepare_snapshot_export "${@:2}"- copy_snapshot_files "${destPath}/${snapshotName}/copylistings"- hdfs dfs -rm -r -f -skipTrash "${destPath}/${snapshotName}/copylistings" > /dev/null- else- copy_snapshot_files "${sourcePath}/copylistings"- echo "Copying the collection meta-data to ${destPath}/${snapshotName}"- ${distCpCmd} "${sourcePath}/${snapshotName}/*" "${destPath}/${snapshotName}/" > /dev/null- fi-- echo "Done. GoodBye!"- ;;- --help)- usage 1>&2- ;;- *)- echo "Unknown command ${cmd}"- usage 1>&2- exit 1-esac