Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-13608: Incremental backup file format (#2250)

Jason Gerlowski33d16b570c76
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java
index 61be096277a..8928fbedcaf 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java
@@ -22,15 +22,13 @@ import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
import static org.apache.solr.common.params.CommonParams.NAME;
+import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.URI;
-import java.time.Instant;
import java.util.Collection;
import java.util.Collections;
import java.util.Optional;
-import java.util.Properties;
-import org.apache.lucene.util.Version;
import org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.ShardRequestTracker;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
@@ -47,11 +45,14 @@ import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.backup.BackupManager;
+import org.apache.solr.core.backup.BackupProperties;
+import org.apache.solr.core.backup.ShardBackupId;
import org.apache.solr.core.backup.repository.BackupRepository;
import org.apache.solr.core.snapshots.CollectionSnapshotMetaData;
import org.apache.solr.core.snapshots.CollectionSnapshotMetaData.CoreSnapshotMetaData;
import org.apache.solr.core.snapshots.CollectionSnapshotMetaData.SnapshotStatus;
import org.apache.solr.core.snapshots.SolrSnapshotManager;
+import org.apache.solr.core.backup.BackupFilePaths;
import org.apache.solr.handler.component.ShardHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,7 +67,9 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
}
@Override
- public void call(ClusterState state, ZkNodeProps message, @SuppressWarnings({"rawtypes"})NamedList results) throws Exception {
+ @SuppressWarnings({"unchecked"})
+ public void call(ClusterState state, ZkNodeProps message, @SuppressWarnings({"rawtypes"}) NamedList results) throws Exception {
+
String extCollectionName = message.getStr(COLLECTION_PROP);
boolean followAliases = message.getBool(FOLLOW_ALIASES, false);
String collectionName;
@@ -77,64 +80,100 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
}
String backupName = message.getStr(NAME);
String repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY);
+ boolean incremental = message.getBool(CoreAdminParams.BACKUP_INCREMENTAL, true);
+ String configName = ocmh.zkStateReader.readConfigName(collectionName);
- Instant startTime = Instant.now();
+ BackupProperties backupProperties = BackupProperties.create(backupName, collectionName,
+ extCollectionName, configName);
CoreContainer cc = ocmh.overseer.getCoreContainer();
- BackupRepository repository = cc.newBackupRepository(repo);
- BackupManager backupMgr = new BackupManager(repository, ocmh.zkStateReader);
+ try (BackupRepository repository = cc.newBackupRepository(repo)) {
+
+ // Backup location
+ URI location = repository.createURI(message.getStr(CoreAdminParams.BACKUP_LOCATION));
+ final URI backupPath = createAndValidateBackupPath(repository, incremental, location, backupName, collectionName);
+
+ BackupManager backupMgr = (incremental) ?
+ BackupManager.forIncrementalBackup(repository, ocmh.zkStateReader, backupPath) :
+ BackupManager.forBackup(repository, ocmh.zkStateReader, backupPath);
+
+ String strategy = message.getStr(CollectionAdminParams.INDEX_BACKUP_STRATEGY, CollectionAdminParams.COPY_FILES_STRATEGY);
+ switch (strategy) {
+ case CollectionAdminParams.COPY_FILES_STRATEGY: {
+ if (incremental) {
+ try {
+ incrementalCopyIndexFiles(backupPath, collectionName, message, results, backupProperties, backupMgr);
+ } catch (SolrException e) {
+ log.error("Error happened during incremental backup for collection:{}", collectionName, e);
+ ocmh.cleanBackup(repository, backupPath, backupMgr.getBackupId());
+ throw e;
+ }
+ } else {
+ copyIndexFiles(backupPath, collectionName, message, results);
+ }
+ break;
+ }
+ case CollectionAdminParams.NO_INDEX_BACKUP_STRATEGY: {
+ break;
+ }
+ }
- // Backup location
- URI location = repository.createURI(message.getStr(CoreAdminParams.BACKUP_LOCATION));
- URI backupPath = repository.resolve(location, backupName);
+ log.info("Starting to backup ZK data for backupName={}", backupName);
- //Validating if the directory already exists.
- if (repository.exists(backupPath)) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The backup directory already exists: " + backupPath);
- }
+ //Download the configs
+ backupMgr.downloadConfigDir(configName);
- // Create a directory to store backup details.
- repository.createDirectory(backupPath);
+ //Save the collection's state. Can be part of the monolithic clusterstate.json or a individual state.json
+ //Since we don't want to distinguish we extract the state and back it up as a separate json
+ DocCollection collectionState = ocmh.zkStateReader.getClusterState().getCollection(collectionName);
+ backupMgr.writeCollectionState(collectionName, collectionState);
+ backupMgr.downloadCollectionProperties(collectionName);
- String strategy = message.getStr(CollectionAdminParams.INDEX_BACKUP_STRATEGY, CollectionAdminParams.COPY_FILES_STRATEGY);
- switch (strategy) {
- case CollectionAdminParams.COPY_FILES_STRATEGY: {
- copyIndexFiles(backupPath, collectionName, message, results);
- break;
- }
- case CollectionAdminParams.NO_INDEX_BACKUP_STRATEGY: {
- break;
- }
- }
+ //TODO: Add MD5 of the configset. If during restore the same name configset exists then we can compare checksums to see if they are the same.
+ //if they are not the same then we can throw an error or have an 'overwriteConfig' flag
+ //TODO save numDocs for the shardLeader. We can use it to sanity check the restore.
- log.info("Starting to backup ZK data for backupName={}", backupName);
+ backupMgr.writeBackupProperties(backupProperties);
- //Download the configs
- String configName = ocmh.zkStateReader.readConfigName(collectionName);
- backupMgr.downloadConfigDir(location, backupName, configName);
+ log.info("Completed backing up ZK data for backupName={}", backupName);
- //Save the collection's state (coming from the collection's state.json)
- //We extract the state and back it up as a separate json
- DocCollection collectionState = ocmh.zkStateReader.getClusterState().getCollection(collectionName);
- backupMgr.writeCollectionState(location, backupName, collectionName, collectionState);
+ int maxNumBackup = message.getInt(CoreAdminParams.MAX_NUM_BACKUP_POINTS, -1);
+ if (incremental && maxNumBackup != -1) {
+ ocmh.deleteBackup(repository, backupPath, maxNumBackup, results);
+ }
+ }
+ }
- Properties properties = new Properties();
+ private URI createAndValidateBackupPath(BackupRepository repository, boolean incremental, URI location, String backupName, String collection) throws IOException{
+ final URI backupNamePath = repository.resolve(location, backupName);
- properties.put(BackupManager.BACKUP_NAME_PROP, backupName);
- properties.put(BackupManager.COLLECTION_NAME_PROP, collectionName);
- properties.put(BackupManager.COLLECTION_ALIAS_PROP, extCollectionName);
- properties.put(CollectionAdminParams.COLL_CONF, configName);
- properties.put(BackupManager.START_TIME_PROP, startTime.toString());
- properties.put(BackupManager.INDEX_VERSION_PROP, Version.LATEST.toString());
- //TODO: Add MD5 of the configset. If during restore the same name configset exists then we can compare checksums to see if they are the same.
- //if they are not the same then we can throw an error or have an 'overwriteConfig' flag
- //TODO save numDocs for the shardLeader. We can use it to sanity check the restore.
+ if ( (!incremental) && repository.exists(backupNamePath)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The backup directory already exists: " + backupNamePath);
+ }
- backupMgr.writeBackupProperties(location, backupName, properties);
+ if (! repository.exists(backupNamePath)) {
+ repository.createDirectory(backupNamePath);
+ } else if (incremental){
+ final String[] directoryContents = repository.listAll(backupNamePath);
+ if (directoryContents.length == 1 && !directoryContents[0].equals(collection)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "The backup [" + backupName + "] at location [" + location +
+ "] cannot be used to back up [" + collection + "], as it already holds a different collection [" +
+ directoryContents[0] + "]");
+ }
+ }
- backupMgr.downloadCollectionProperties(location, backupName, collectionName);
+ if (! incremental) {
+ return backupNamePath;
+ }
- log.info("Completed backing up ZK data for backupName={}", backupName);
+ // Incremental backups have an additional directory named after the collection that needs created
+ final URI backupPathWithCollection = repository.resolve(backupNamePath, collection);
+ if (! repository.exists(backupPathWithCollection)) {
+ repository.createDirectory(backupPathWithCollection);
+ }
+ BackupFilePaths incBackupFiles = new BackupFilePaths(repository, backupPathWithCollection);
+ incBackupFiles.createIncrementalBackupFolders();
+ return backupPathWithCollection;
}
private Replica selectReplicaWithSnapshot(CollectionSnapshotMetaData snapshotMeta, Slice slice) {
@@ -142,9 +181,9 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
// If that is not possible, we choose any other replica for the given shard.
Collection<CoreSnapshotMetaData> snapshots = snapshotMeta.getReplicaSnapshotsForShard(slice.getName());
- Optional<CoreSnapshotMetaData> leaderCore = snapshots.stream().filter(x -> x.isLeader()).findFirst();
+ Optional<CoreSnapshotMetaData> leaderCore = snapshots.stream().filter(CoreSnapshotMetaData::isLeader).findFirst();
if (leaderCore.isPresent()) {
- if (log.isInfoEnabled()) {
+ if (log.isInfoEnabled()) {
log.info("Replica {} was the leader when snapshot {} was created.", leaderCore.get().getCoreName(), snapshotMeta.getName());
}
Replica r = slice.getReplica(leaderCore.get().getCoreName());
@@ -154,19 +193,103 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
}
Optional<Replica> r = slice.getReplicas().stream()
- .filter(x -> x.getState() != State.DOWN && snapshotMeta.isSnapshotExists(slice.getName(), x))
- .findFirst();
+ .filter(x -> x.getState() != State.DOWN && snapshotMeta.isSnapshotExists(slice.getName(), x))
+ .findFirst();
if (!r.isPresent()) {
throw new SolrException(ErrorCode.SERVER_ERROR,
- "Unable to find any live replica with a snapshot named " + snapshotMeta.getName() + " for shard " + slice.getName());
+ "Unable to find any live replica with a snapshot named " + snapshotMeta.getName() + " for shard " + slice.getName());
}
return r.get();
}
+ private void incrementalCopyIndexFiles(URI backupPath, String collectionName, ZkNodeProps request,
+ NamedList<Object> results, BackupProperties backupProperties,
+ BackupManager backupManager) throws IOException {
+ String backupName = request.getStr(NAME);
+ String asyncId = request.getStr(ASYNC);
+ String repoName = request.getStr(CoreAdminParams.BACKUP_REPOSITORY);
+ ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler();
+
+ log.info("Starting backup of collection={} with backupName={} at location={}", collectionName, backupName,
+ backupPath);
+
+ Optional<BackupProperties> previousProps = backupManager.tryReadBackupProperties();
+ final ShardRequestTracker shardRequestTracker = ocmh.asyncRequestTracker(asyncId);
+
+ Collection<Slice> slices = ocmh.zkStateReader.getClusterState().getCollection(collectionName).getActiveSlices();
+ for (Slice slice : slices) {
+ // Note - Actually this can return a null value when there is no leader for this shard.
+ Replica replica = slice.getLeader();
+ if (replica == null) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "No 'leader' replica available for shard " + slice.getName() + " of collection " + collectionName);
+ }
+ String coreName = replica.getStr(CORE_NAME_PROP);
+
+ ModifiableSolrParams params = coreBackupParams(backupPath, repoName, slice, coreName, true /* incremental backup */);
+ params.set(CoreAdminParams.BACKUP_INCREMENTAL, true);
+ previousProps.flatMap(bp -> bp.getShardBackupIdFor(slice.getName()))
+ .ifPresent(prevBackupPoint -> params.set(CoreAdminParams.PREV_SHARD_BACKUP_ID, prevBackupPoint.getIdAsString()));
+
+ ShardBackupId shardBackupId = backupProperties.putAndGetShardBackupIdFor(slice.getName(),
+ backupManager.getBackupId().getId());
+ params.set(CoreAdminParams.SHARD_BACKUP_ID, shardBackupId.getIdAsString());
+
+ shardRequestTracker.sendShardRequest(replica.getNodeName(), params, shardHandler);
+ log.debug("Sent backup request to core={} for backupName={}", coreName, backupName);
+ }
+ log.debug("Sent backup requests to all shard leaders for backupName={}", backupName);
+
+ String msgOnError = "Could not backup all shards";
+ shardRequestTracker.processResponses(results, shardHandler, true, msgOnError);
+ if (results.get("failure") != null) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, msgOnError);
+ }
+
+ //Aggregating result from different shards
+ @SuppressWarnings({"rawtypes"})
+ NamedList aggRsp = aggregateResults(results, collectionName, backupManager, backupProperties, slices);
+ results.add("response", aggRsp);
+ }
+
+ @SuppressWarnings({"rawtypes"})
+ private NamedList aggregateResults(NamedList results, String collectionName,
+ BackupManager backupManager,
+ BackupProperties backupProps,
+ Collection<Slice> slices) {
+ NamedList<Object> aggRsp = new NamedList<>();
+ aggRsp.add("collection", collectionName);
+ aggRsp.add("numShards", slices.size());
+ aggRsp.add("backupId", backupManager.getBackupId().id);
+ aggRsp.add("indexVersion", backupProps.getIndexVersion());
+ aggRsp.add("startTime", backupProps.getStartTime());
+
+ double indexSizeMB = 0;
+ NamedList shards = (NamedList) results.get("success");
+ for (int i = 0; i < shards.size(); i++) {
+ NamedList shardResp = (NamedList)((NamedList)shards.getVal(i)).get("response");
+ if (shardResp == null)
+ continue;
+ indexSizeMB += (double) shardResp.get("indexSizeMB");
+ }
+ aggRsp.add("indexSizeMB", indexSizeMB);
+ return aggRsp;
+ }
+
+ private ModifiableSolrParams coreBackupParams(URI backupPath, String repoName, Slice slice, String coreName, boolean incremental) {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString());
+ params.set(NAME, slice.getName());
+ params.set(CoreAdminParams.BACKUP_REPOSITORY, repoName);
+ params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString()); // note: index dir will be here then the "snapshot." + slice name
+ params.set(CORE_NAME_PROP, coreName);
+ params.set(CoreAdminParams.BACKUP_INCREMENTAL, incremental);
+ return params;
+ }
+
@SuppressWarnings({"unchecked"})
- private void copyIndexFiles(URI backupPath, String collectionName, ZkNodeProps request, @SuppressWarnings({"rawtypes"})NamedList results) throws Exception {
+ private void copyIndexFiles(URI backupPath, String collectionName, ZkNodeProps request, @SuppressWarnings({"rawtypes"}) NamedList results) throws Exception {
String backupName = request.getStr(NAME);
String asyncId = request.getStr(ASYNC);
String repoName = request.getStr(CoreAdminParams.BACKUP_REPOSITORY);
@@ -179,16 +302,16 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
snapshotMeta = SolrSnapshotManager.getCollectionLevelSnapshot(zkClient, collectionName, commitName);
if (!snapshotMeta.isPresent()) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName
- + " does not exist for collection " + collectionName);
+ + " does not exist for collection " + collectionName);
}
if (snapshotMeta.get().getStatus() != SnapshotStatus.Successful) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName + " for collection " + collectionName
- + " has not completed successfully. The status is " + snapshotMeta.get().getStatus());
+ + " has not completed successfully. The status is " + snapshotMeta.get().getStatus());
}
}
log.info("Starting backup of collection={} with backupName={} at location={}", collectionName, backupName,
- backupPath);
+ backupPath);
Collection<String> shardsToConsider = Collections.emptySet();
if (snapshotMeta.isPresent()) {
@@ -202,7 +325,7 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
if (snapshotMeta.isPresent()) {
if (!shardsToConsider.contains(slice.getName())) {
log.warn("Skipping the backup for shard {} since it wasn't part of the collection {} when snapshot {} was created.",
- slice.getName(), collectionName, snapshotMeta.get().getName());
+ slice.getName(), collectionName, snapshotMeta.get().getName());
continue;
}
replica = selectReplicaWithSnapshot(snapshotMeta.get(), slice);
@@ -216,12 +339,7 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
String coreName = replica.getStr(CORE_NAME_PROP);
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString());
- params.set(NAME, slice.getName());
- params.set(CoreAdminParams.BACKUP_REPOSITORY, repoName);
- params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString()); // note: index dir will be here then the "snapshot." + slice name
- params.set(CORE_NAME_PROP, coreName);
+ ModifiableSolrParams params = coreBackupParams(backupPath, repoName, slice, coreName, false /*non-incremental backup */);
if (snapshotMeta.isPresent()) {
params.set(CoreAdminParams.COMMIT_NAME, snapshotMeta.get().getName());
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java
new file mode 100644
index 00000000000..7709751d640
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.api.collections;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.params.CoreAdminParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.backup.BackupId;
+import org.apache.solr.core.backup.AggregateBackupStats;
+import org.apache.solr.core.backup.BackupManager;
+import org.apache.solr.core.backup.BackupProperties;
+import org.apache.solr.core.backup.ShardBackupId;
+import org.apache.solr.core.backup.ShardBackupMetadata;
+import org.apache.solr.core.backup.repository.BackupRepository;
+import org.apache.solr.core.backup.BackupFilePaths;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.apache.solr.common.params.CommonParams.NAME;
+import static org.apache.solr.core.backup.BackupManager.COLLECTION_NAME_PROP;
+import static org.apache.solr.core.backup.BackupManager.START_TIME_PROP;
+
+/**
+ * An overseer command used to delete files associated with incremental backups.
+ *
+ * This assumes use of the incremental backup format, and not the (now deprecated) traditional 'full-snapshot' format.
+ * The deletion can either delete a specific {@link BackupId}, or delete everything except the most recent N backup
+ * points.
+ *
+ * While this functionality is structured as an overseer command, the message type isn't currently handled by the
+ * overseer or recorded to the overseer queue by any APIs. This integration will be added in the forthcoming SOLR-15101
+ */
+public class DeleteBackupCmd implements OverseerCollectionMessageHandler.Cmd {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final OverseerCollectionMessageHandler ocmh;
+
+ DeleteBackupCmd(OverseerCollectionMessageHandler ocmh) {
+ this.ocmh = ocmh;
+ }
+
+ @Override
+ public void call(ClusterState state, ZkNodeProps message, @SuppressWarnings({"rawtypes"}) NamedList results) throws Exception {
+ String backupLocation = message.getStr(CoreAdminParams.BACKUP_LOCATION);
+ String backupName = message.getStr(NAME);
+ String repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY);
+ int backupId = message.getInt(CoreAdminParams.BACKUP_ID, -1);
+ int lastNumBackupPointsToKeep = message.getInt(CoreAdminParams.MAX_NUM_BACKUP_POINTS, -1);
+ if (backupId == -1 && lastNumBackupPointsToKeep == -1) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ String.format(Locale.ROOT, "%s or %s param must be provided", CoreAdminParams.BACKUP_ID, CoreAdminParams.MAX_NUM_BACKUP_POINTS));
+ }
+ CoreContainer cc = ocmh.overseer.getCoreContainer();
+ try (BackupRepository repository = cc.newBackupRepository(repo)) {
+ URI location = repository.createURI(backupLocation);
+ final URI backupPath = BackupFilePaths.buildExistingBackupLocationURI(repository, location, backupName);
+ if (repository.exists(repository.resolve(backupPath, BackupManager.TRADITIONAL_BACKUP_PROPS_FILE))) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The backup name [" + backupName + "] at " +
+ "location [" + location + "] holds a non-incremental (legacy) backup, but " +
+ "backup-deletion is only supported on incremental backups");
+ }
+
+ if (backupId != -1){
+ deleteBackupId(repository, backupPath, backupId, results);
+ } else {
+ keepNumberOfBackup(repository, backupPath, lastNumBackupPointsToKeep, results);
+ }
+ }
+ }
+
+ /**
+ * Keep most recent {@code maxNumBackup} and delete the rest.
+ */
+ void keepNumberOfBackup(BackupRepository repository, URI backupPath,
+ int maxNumBackup,
+ @SuppressWarnings({"rawtypes"}) NamedList results) throws Exception {
+ List<BackupId> backupIds = BackupFilePaths.findAllBackupIdsFromFileListing(repository.listAllOrEmpty(backupPath));
+ if (backupIds.size() <= maxNumBackup) {
+ return;
+ }
+
+ Collections.sort(backupIds);
+ List<BackupId> backupIdDeletes = backupIds.subList(0, backupIds.size() - maxNumBackup);
+ deleteBackupIds(backupPath, repository, new HashSet<>(backupIdDeletes), results);
+ }
+
+ void deleteBackupIds(URI backupPath, BackupRepository repository,
+ Set<BackupId> backupIdsDeletes,
+ @SuppressWarnings({"rawtypes"}) NamedList results) throws IOException {
+ BackupFilePaths incBackupFiles = new BackupFilePaths(repository, backupPath);
+ URI shardBackupMetadataDir = incBackupFiles.getShardBackupMetadataDir();
+
+ Set<String> referencedIndexFiles = new HashSet<>();
+ List<ShardBackupId> shardBackupIdFileDeletes = new ArrayList<>();
+
+
+ List<ShardBackupId> shardBackupIds = Arrays.stream(repository.listAllOrEmpty(shardBackupMetadataDir))
+ .map(sbi -> ShardBackupId.fromShardMetadataFilename(sbi))
+ .collect(Collectors.toList());
+ for (ShardBackupId shardBackupId : shardBackupIds) {
+ final BackupId backupId = shardBackupId.getContainingBackupId();
+
+ if (backupIdsDeletes.contains(backupId)) {
+ shardBackupIdFileDeletes.add(shardBackupId);
+ } else {
+ ShardBackupMetadata shardBackupMetadata = ShardBackupMetadata.from(repository, shardBackupMetadataDir, shardBackupId);
+ if (shardBackupMetadata != null)
+ referencedIndexFiles.addAll(shardBackupMetadata.listUniqueFileNames());
+ }
+ }
+
+
+ Map<BackupId, AggregateBackupStats> backupIdToCollectionBackupPoint = new HashMap<>();
+ List<String> unusedFiles = new ArrayList<>();
+ for (ShardBackupId shardBackupIdToDelete : shardBackupIdFileDeletes) {
+ BackupId backupId = shardBackupIdToDelete.getContainingBackupId();
+ ShardBackupMetadata shardBackupMetadata = ShardBackupMetadata.from(repository, shardBackupMetadataDir, shardBackupIdToDelete);
+ if (shardBackupMetadata == null)
+ continue;
+
+ backupIdToCollectionBackupPoint
+ .putIfAbsent(backupId, new AggregateBackupStats());
+ backupIdToCollectionBackupPoint.get(backupId).add(shardBackupMetadata);
+
+ for (String uniqueIndexFile : shardBackupMetadata.listUniqueFileNames()) {
+ if (!referencedIndexFiles.contains(uniqueIndexFile)) {
+ unusedFiles.add(uniqueIndexFile);
+ }
+ }
+ }
+
+ repository.delete(incBackupFiles.getShardBackupMetadataDir(),
+ shardBackupIdFileDeletes.stream().map(ShardBackupId::getIdAsString).collect(Collectors.toList()), true);
+ repository.delete(incBackupFiles.getIndexDir(), unusedFiles, true);
+ try {
+ for (BackupId backupId : backupIdsDeletes) {
+ repository.deleteDirectory(repository.resolve(backupPath, BackupFilePaths.getZkStateDir(backupId)));
+ }
+ } catch (FileNotFoundException e) {
+ //ignore this
+ }
+
+ //add details to result before deleting backupPropFiles
+ addResult(backupPath, repository, backupIdsDeletes, backupIdToCollectionBackupPoint, results);
+ repository.delete(backupPath, backupIdsDeletes.stream().map(id -> BackupFilePaths.getBackupPropsName(id)).collect(Collectors.toList()), true);
+ }
+
+ @SuppressWarnings("unchecked")
+ private void addResult(URI backupPath, BackupRepository repository,
+ Set<BackupId> backupIdDeletes,
+ Map<BackupId, AggregateBackupStats> backupIdToCollectionBackupPoint,
+ @SuppressWarnings({"rawtypes"}) NamedList results) {
+
+ String collectionName = null;
+ @SuppressWarnings({"rawtypes"})
+ List<NamedList> shardBackupIdDetails = new ArrayList<>();
+ results.add("deleted", shardBackupIdDetails);
+ for (BackupId backupId : backupIdDeletes) {
+ NamedList<Object> backupIdResult = new NamedList<>();
+
+ try {
+ BackupProperties props = BackupProperties.readFrom(repository, backupPath, BackupFilePaths.getBackupPropsName(backupId));
+ backupIdResult.add(START_TIME_PROP, props.getStartTime());
+ if (collectionName == null) {
+ collectionName = props.getCollection();
+ results.add(COLLECTION_NAME_PROP, collectionName);
+ }
+ } catch (IOException e) {
+ //prop file not found
+ }
+
+ AggregateBackupStats cbp = backupIdToCollectionBackupPoint.getOrDefault(backupId, new AggregateBackupStats());
+ backupIdResult.add("backupId", backupId.getId());
+ backupIdResult.add("size", cbp.getTotalSize());
+ backupIdResult.add("numFiles", cbp.getNumFiles());
+ shardBackupIdDetails.add(backupIdResult);
+ }
+ }
+
+ private void deleteBackupId(BackupRepository repository, URI backupPath,
+ int bid, @SuppressWarnings({"rawtypes"}) NamedList results) throws Exception {
+ BackupId backupId = new BackupId(bid);
+ if (!repository.exists(repository.resolve(backupPath, BackupFilePaths.getBackupPropsName(backupId)))) {
+ return;
+ }
+
+ deleteBackupIds(backupPath, repository, Collections.singleton(backupId), results);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
index 23e677711e9..b7f9c6d076a 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
@@ -18,6 +18,7 @@ package org.apache.solr.cloud.api.collections;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -82,6 +83,8 @@ import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.SuppressForbidden;
import org.apache.solr.common.util.TimeSource;
import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.backup.BackupId;
+import org.apache.solr.core.backup.repository.BackupRepository;
import org.apache.solr.handler.component.HttpShardHandlerFactory;
import org.apache.solr.handler.component.ShardHandler;
import org.apache.solr.handler.component.ShardRequest;
@@ -200,6 +203,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
.put(DELETENODE, new DeleteNodeCmd(this))
.put(BACKUP, new BackupCmd(this))
.put(RESTORE, new RestoreCmd(this))
+ .put(DELETEBACKUP, new DeleteBackupCmd(this))
.put(CREATESNAPSHOT, new CreateSnapshotCmd(this))
.put(DELETESNAPSHOT, new DeleteSnapshotCmd(this))
.put(SPLITSHARD, new SplitShardCmd(this))
@@ -625,6 +629,19 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
return results;
}
+ @SuppressWarnings({"rawtypes"})
+ void cleanBackup(BackupRepository repository, URI backupPath, BackupId backupId) throws Exception {
+ ((DeleteBackupCmd)commandMap.get(DELETEBACKUP))
+ .deleteBackupIds(backupPath, repository, Collections.singleton(backupId), new NamedList());
+ }
+
+ void deleteBackup(BackupRepository repository, URI backupPath,
+ int maxNumBackup,
+ @SuppressWarnings({"rawtypes"}) NamedList results) throws Exception {
+ ((DeleteBackupCmd)commandMap.get(DELETEBACKUP))
+ .keepNumberOfBackup(repository, backupPath, maxNumBackup, results);
+ }
+
List<ZkNodeProps> addReplica(ClusterState clusterState, ZkNodeProps message, @SuppressWarnings({"rawtypes"})NamedList results, Runnable onComplete)
throws Exception {
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
index 03cb47a6252..48a584bbb4f 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
@@ -18,22 +18,6 @@
package org.apache.solr.cloud.api.collections;
-import java.lang.invoke.MethodHandles;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
import org.apache.solr.cloud.Overseer;
import org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.ShardRequestTracker;
import org.apache.solr.cloud.overseer.OverseerAction;
@@ -56,18 +40,34 @@ import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.backup.BackupManager;
+import org.apache.solr.core.backup.BackupProperties;
+import org.apache.solr.core.backup.ShardBackupId;
import org.apache.solr.core.backup.repository.BackupRepository;
import org.apache.solr.handler.component.ShardHandler;
+import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS;
-import static org.apache.solr.common.cloud.ZkStateReader.PULL_REPLICAS;
-import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
-import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE;
-import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.TLOG_REPLICAS;
+import java.io.Closeable;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.solr.common.cloud.ZkStateReader.*;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESHARD;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
@@ -85,72 +85,209 @@ public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd {
@Override
@SuppressWarnings({"unchecked", "rawtypes"})
public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {
- // TODO maybe we can inherit createCollection's options/code
-
- String restoreCollectionName = message.getStr(COLLECTION_PROP);
- String backupName = message.getStr(NAME); // of backup
- ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler();
- String asyncId = message.getStr(ASYNC);
- String repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY);
-
- CoreContainer cc = ocmh.overseer.getCoreContainer();
- BackupRepository repository = cc.newBackupRepository(repo);
-
- URI location = repository.createURI(message.getStr(CoreAdminParams.BACKUP_LOCATION));
- URI backupPath = repository.resolve(location, backupName);
- ZkStateReader zkStateReader = ocmh.zkStateReader;
- BackupManager backupMgr = new BackupManager(repository, zkStateReader);
-
- Properties properties = backupMgr.readBackupProperties(location, backupName);
- String backupCollection = properties.getProperty(BackupManager.COLLECTION_NAME_PROP);
-
- // Test if the collection is of stateFormat 1 (i.e. not 2) supported pre Solr 9, in which case can't restore it.
- Object format = properties.get("stateFormat");
- if (format != null && !"2".equals(format)) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "Collection " + backupCollection + " is in stateFormat=" + format +
- " no longer supported in Solr 9 and above. It can't be restored. If it originates in Solr 8 you can restore" +
- " it there, migrate it to stateFormat=2 and backup again, it will then be restorable on Solr 9");
+ try (RestoreContext restoreContext = new RestoreContext(message, ocmh)) {
+ if (state.hasCollection(restoreContext.restoreCollectionName)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Restoration collection [" + restoreContext.restoreCollectionName +
+ "] must be created by the backup process and cannot exist");
+ } else {
+ RestoreOnANewCollection restoreOnANewCollection = new RestoreOnANewCollection(message, restoreContext.backupCollectionState);
+ restoreOnANewCollection.validate(restoreContext.backupCollectionState, restoreContext.nodeList.size());
+ restoreOnANewCollection.process(results, restoreContext);
+ }
+ }
+ }
+
+ private void requestReplicasToRestore(@SuppressWarnings({"rawtypes"}) NamedList results, DocCollection restoreCollection,
+ ClusterState clusterState,
+ BackupProperties backupProperties,
+ URI backupPath,
+ String repo,
+ ShardHandler shardHandler,
+ String asyncId) {
+ ShardRequestTracker shardRequestTracker = ocmh.asyncRequestTracker(asyncId);
+ // Copy data from backed up index to each replica
+ for (Slice slice : restoreCollection.getSlices()) {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.RESTORECORE.toString());
+ Optional<ShardBackupId> shardBackupId = backupProperties.getShardBackupIdFor(slice.getName());
+ if (shardBackupId.isPresent()) {
+ params.set(CoreAdminParams.SHARD_BACKUP_ID, shardBackupId.get().getIdAsString());
+ } else {
+ params.set(NAME, "snapshot." + slice.getName());
+ }
+ params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString());
+ params.set(CoreAdminParams.BACKUP_REPOSITORY, repo);
+ shardRequestTracker.sliceCmd(clusterState, params, null, slice, shardHandler);
+ }
+ shardRequestTracker.processResponses(new NamedList<>(), shardHandler, true, "Could not restore core");
+ }
+
+ private int getInt(ZkNodeProps message, String propertyName, Integer count, int defaultValue) {
+ Integer value = message.getInt(propertyName, count);
+ return value!=null ? value:defaultValue;
+ }
+
+ /**
+ * Encapsulates the parsing and access for common parameters restore parameters and values
+ */
+ private static class RestoreContext implements Closeable {
+
+ final String restoreCollectionName;
+ final String backupName;
+ final String backupCollection;
+ final String asyncId;
+ final String repo;
+ final String restoreConfigName;
+ final int backupId;
+ final URI location;
+ final URI backupPath;
+ final List<String> nodeList;
+
+ final CoreContainer container;
+ final BackupRepository repository;
+ final ZkStateReader zkStateReader;
+ final BackupManager backupManager;
+ final BackupProperties backupProperties;
+ final DocCollection backupCollectionState;
+ final ShardHandler shardHandler;
+
+ private RestoreContext(ZkNodeProps message, OverseerCollectionMessageHandler ocmh) throws IOException {
+ this.restoreCollectionName = message.getStr(COLLECTION_PROP);
+ this.backupName = message.getStr(NAME); // of backup
+ this.asyncId = message.getStr(ASYNC);
+ this.repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY);
+ this.backupId = message.getInt(CoreAdminParams.BACKUP_ID, -1);
+
+ this.container = ocmh.overseer.getCoreContainer();
+ this.repository = this.container.newBackupRepository(repo);
+
+ this.location = repository.createURI(message.getStr(CoreAdminParams.BACKUP_LOCATION));
+ final URI backupNameUri = repository.resolve(location, backupName);
+ final String[] entries = repository.listAll(backupNameUri);
+ final boolean incremental = ! Arrays.stream(entries).anyMatch(entry -> entry.equals(BackupManager.TRADITIONAL_BACKUP_PROPS_FILE));
+ this.backupPath = (incremental) ?
+ repository.resolve(backupNameUri, entries[0]) : // incremental backups have an extra path component representing the backed up collection
+ backupNameUri;
+ this.zkStateReader = ocmh.zkStateReader;
+ this.backupManager = backupId == -1 ?
+ BackupManager.forRestore(repository, zkStateReader, backupPath) :
+ BackupManager.forRestore(repository, zkStateReader, backupPath, backupId);
+
+ this.backupProperties = this.backupManager.readBackupProperties();
+ this.backupCollection = this.backupProperties.getCollection();
+ this.restoreConfigName = message.getStr(CollectionAdminParams.COLL_CONF, this.backupProperties.getConfigName());
+ this.backupCollectionState = this.backupManager.readCollectionState(this.backupCollection);
+
+ this.shardHandler = ocmh.shardHandlerFactory.getShardHandler();
+ this.nodeList = Assign.getLiveOrLiveAndCreateNodeSetList(
+ zkStateReader.getClusterState().getLiveNodes(), message, OverseerCollectionMessageHandler.RANDOM);
}
- String backupCollectionAlias = properties.getProperty(BackupManager.COLLECTION_ALIAS_PROP);
- DocCollection backupCollectionState = backupMgr.readCollectionState(location, backupName, backupCollection);
-
- // Get the Solr nodes to restore a collection.
- final List<String> nodeList = Assign.getLiveOrLiveAndCreateNodeSetList(
- zkStateReader.getClusterState().getLiveNodes(), message, OverseerCollectionMessageHandler.RANDOM);
-
- int numShards = backupCollectionState.getActiveSlices().size();
-
- int numNrtReplicas;
- if (message.get(REPLICATION_FACTOR) != null) {
- numNrtReplicas = message.getInt(REPLICATION_FACTOR, 0);
- } else if (message.get(NRT_REPLICAS) != null) {
- numNrtReplicas = message.getInt(NRT_REPLICAS, 0);
- } else {
- //replicationFactor and nrtReplicas is always in sync after SOLR-11676
- //pick from cluster state of the backed up collection
- numNrtReplicas = backupCollectionState.getReplicationFactor();
+
+ @Override
+ public void close() throws IOException {
+ if (this.repository != null) {
+ this.repository.close();
+ }
+ }
+ }
+
+ /**
+ * Restoration 'strategy' that takes responsibility for creating the collection to restore to.
+ *
+ * This is currently the only supported 'strategy' for backup restoration. Though in-place restoration has been
+ * proposed and may be added soon (see SOLR-15087)
+ */
+ private class RestoreOnANewCollection {
+ private int numNrtReplicas;
+ private int numTlogReplicas;
+ private int numPullReplicas;
+ private ZkNodeProps message;
+
+ private RestoreOnANewCollection(ZkNodeProps message, DocCollection backupCollectionState) {
+ this.message = message;
+
+ if (message.get(REPLICATION_FACTOR) != null) {
+ this.numNrtReplicas = message.getInt(REPLICATION_FACTOR, 0);
+ } else if (message.get(NRT_REPLICAS) != null) {
+ this.numNrtReplicas = message.getInt(NRT_REPLICAS, 0);
+ } else {
+ //replicationFactor and nrtReplicas is always in sync after SOLR-11676
+ //pick from cluster state of the backed up collection
+ this.numNrtReplicas = backupCollectionState.getReplicationFactor();
+ }
+ this.numTlogReplicas = getInt(message, TLOG_REPLICAS, backupCollectionState.getNumTlogReplicas(), 0);
+ this.numPullReplicas = getInt(message, PULL_REPLICAS, backupCollectionState.getNumPullReplicas(), 0);
+ }
+
+ public void process(@SuppressWarnings("rawtypes") NamedList results, RestoreContext rc) throws Exception {
+ // Avoiding passing RestoreContext around
+ uploadConfig(rc.backupProperties.getConfigName(),
+ rc.restoreConfigName,
+ rc.zkStateReader,
+ rc.backupManager);
+
+ log.info("Starting restore into collection={} with backup_name={} at location={}", rc.restoreCollectionName, rc.backupName,
+ rc.location);
+ createCoreLessCollection(rc.restoreCollectionName,
+ rc.restoreConfigName,
+ rc.backupCollectionState,
+ rc.zkStateReader.getClusterState());
+ // note: when createCollection() returns, the collection exists (no race)
+
+ // Restore collection properties
+ rc.backupManager.uploadCollectionProperties(rc.restoreCollectionName);
+
+ DocCollection restoreCollection = rc.zkStateReader.getClusterState().getCollection(rc.restoreCollectionName);
+ markAllShardsAsConstruction(restoreCollection);
+ // TODO how do we leverage the RULE / SNITCH logic in createCollection?
+ ClusterState clusterState = rc.zkStateReader.getClusterState();
+
+ List<String> sliceNames = new ArrayList<>();
+ restoreCollection.getSlices().forEach(x -> sliceNames.add(x.getName()));
+
+ List<ReplicaPosition> replicaPositions = getReplicaPositions(restoreCollection, rc.nodeList, clusterState, sliceNames);
+
+ createSingleReplicaPerShard(results, restoreCollection, rc.asyncId, clusterState, replicaPositions);
+ Object failures = results.get("failure");
+ if (failures != null && ((SimpleOrderedMap) failures).size() > 0) {
+ log.error("Restore failed to create initial replicas.");
+ ocmh.cleanupCollection(rc.restoreCollectionName, new NamedList<>());
+ return;
+ }
+
+ //refresh the location copy of collection state
+ restoreCollection = rc.zkStateReader.getClusterState().getCollection(rc.restoreCollectionName);
+ requestReplicasToRestore(results, restoreCollection, clusterState, rc.backupProperties, rc.backupPath, rc.repo, rc.shardHandler, rc.asyncId);
+ requestReplicasToApplyBufferUpdates(restoreCollection, rc.asyncId, rc.shardHandler);
+ markAllShardsAsActive(restoreCollection);
+ addReplicasToShards(results, clusterState, restoreCollection, replicaPositions, rc.asyncId);
+ restoringAlias(rc.backupProperties);
+
+ log.info("Completed restoring collection={} backupName={}", restoreCollection, rc.backupName);
+
}
- int numTlogReplicas = getInt(message, TLOG_REPLICAS, backupCollectionState.getNumTlogReplicas(), 0);
- int numPullReplicas = getInt(message, PULL_REPLICAS, backupCollectionState.getNumPullReplicas(), 0);
- int totalReplicasPerShard = numNrtReplicas + numTlogReplicas + numPullReplicas;
- assert totalReplicasPerShard > 0;
-
- //Upload the configs
- String configName = (String) properties.get(CollectionAdminParams.COLL_CONF);
- String restoreConfigName = message.getStr(CollectionAdminParams.COLL_CONF, configName);
- if (zkStateReader.getConfigManager().configExists(restoreConfigName)) {
- log.info("Using existing config {}", restoreConfigName);
- //TODO add overwrite option?
- } else {
- log.info("Uploading config {}", restoreConfigName);
- backupMgr.uploadConfigDir(location, backupName, configName, restoreConfigName);
+
+ private void validate(DocCollection backupCollectionState, int availableNodeCount) {
+ int numShards = backupCollectionState.getActiveSlices().size();
+ int totalReplicasPerShard = numNrtReplicas + numTlogReplicas + numPullReplicas;
+ assert totalReplicasPerShard > 0;
}
- log.info("Starting restore into collection={} with backup_name={} at location={}", restoreCollectionName, backupName,
- location);
+ private void uploadConfig(String configName, String restoreConfigName, ZkStateReader zkStateReader, BackupManager backupMgr) throws IOException {
+ if (zkStateReader.getConfigManager().configExists(restoreConfigName)) {
+ log.info("Using existing config {}", restoreConfigName);
+ //TODO add overwrite option?
+ } else {
+ log.info("Uploading config {}", restoreConfigName);
+ backupMgr.uploadConfigDir(configName, restoreConfigName);
+ }
+ }
- //Create core-less collection
- {
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ private void createCoreLessCollection(String restoreCollectionName,
+ String restoreConfigName,
+ DocCollection backupCollectionState,
+ ClusterState clusterState) throws Exception {
Map<String, Object> propMap = new HashMap<>();
propMap.put(Overseer.QUEUE_OPERATION, CREATE.toString());
propMap.put("fromApi", "true"); // mostly true. Prevents autoCreated=true in the collection state.
@@ -193,135 +330,107 @@ public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd {
propMap.put(OverseerCollectionMessageHandler.SHARDS_PROP, newSlices);
}
- ocmh.commandMap.get(CREATE).call(zkStateReader.getClusterState(), new ZkNodeProps(propMap), new NamedList());
+ ocmh.commandMap.get(CREATE).call(clusterState, new ZkNodeProps(propMap), new NamedList());
// note: when createCollection() returns, the collection exists (no race)
}
- // Restore collection properties
- backupMgr.uploadCollectionProperties(location, backupName, restoreCollectionName);
-
- DocCollection restoreCollection = zkStateReader.getClusterState().getCollection(restoreCollectionName);
-
//Mark all shards in CONSTRUCTION STATE while we restore the data
- {
+ private void markAllShardsAsConstruction(DocCollection restoreCollection) throws KeeperException, InterruptedException {
//TODO might instead createCollection accept an initial state? Is there a race?
Map<String, Object> propMap = new HashMap<>();
propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower());
for (Slice shard : restoreCollection.getSlices()) {
propMap.put(shard.getName(), Slice.State.CONSTRUCTION.toString());
}
- propMap.put(ZkStateReader.COLLECTION_PROP, restoreCollectionName);
+ propMap.put(ZkStateReader.COLLECTION_PROP, restoreCollection.getName());
ocmh.overseer.offerStateUpdate(Utils.toJSON(new ZkNodeProps(propMap)));
}
- // TODO how do we leverage the RULE / SNITCH logic in createCollection?
-
- ClusterState clusterState = zkStateReader.getClusterState();
-
- List<String> sliceNames = new ArrayList<>();
- restoreCollection.getSlices().forEach(x -> sliceNames.add(x.getName()));
-
- Assign.AssignRequest assignRequest = new Assign.AssignRequestBuilder()
- .forCollection(restoreCollectionName)
- .forShard(sliceNames)
- .assignNrtReplicas(numNrtReplicas)
- .assignTlogReplicas(numTlogReplicas)
- .assignPullReplicas(numPullReplicas)
- .onNodes(nodeList)
- .build();
- Assign.AssignStrategy assignStrategy = Assign.createAssignStrategy(
- ocmh.overseer.getCoreContainer(),
- clusterState, restoreCollection);
- List<ReplicaPosition> replicaPositions = assignStrategy.assign(ocmh.cloudManager, assignRequest);
+ private List<ReplicaPosition> getReplicaPositions(DocCollection restoreCollection, List<String> nodeList, ClusterState clusterState, List<String> sliceNames) throws IOException, InterruptedException {
+ Assign.AssignRequest assignRequest = new Assign.AssignRequestBuilder()
+ .forCollection(restoreCollection.getName())
+ .forShard(sliceNames)
+ .assignNrtReplicas(numNrtReplicas)
+ .assignTlogReplicas(numTlogReplicas)
+ .assignPullReplicas(numPullReplicas)
+ .onNodes(nodeList)
+ .build();
+ Assign.AssignStrategy assignStrategy = Assign.createAssignStrategy(
+ ocmh.overseer.getCoreContainer(),
+ clusterState, restoreCollection);
+ return assignStrategy.assign(ocmh.cloudManager, assignRequest);
+ }
- CountDownLatch countDownLatch = new CountDownLatch(restoreCollection.getSlices().size());
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ private void createSingleReplicaPerShard(NamedList results,
+ DocCollection restoreCollection,
+ String asyncId,
+ ClusterState clusterState, List<ReplicaPosition> replicaPositions) throws Exception {
+ CountDownLatch countDownLatch = new CountDownLatch(restoreCollection.getSlices().size());
- //Create one replica per shard and copy backed up data to it
- for (Slice slice : restoreCollection.getSlices()) {
- if (log.isInfoEnabled()) {
- log.info("Adding replica for shard={} collection={} ", slice.getName(), restoreCollection);
- }
- HashMap<String, Object> propMap = new HashMap<>();
- propMap.put(Overseer.QUEUE_OPERATION, CREATESHARD);
- propMap.put(COLLECTION_PROP, restoreCollectionName);
- propMap.put(SHARD_ID_PROP, slice.getName());
-
- if (numNrtReplicas >= 1) {
- propMap.put(REPLICA_TYPE, Replica.Type.NRT.name());
- } else if (numTlogReplicas >= 1) {
- propMap.put(REPLICA_TYPE, Replica.Type.TLOG.name());
- } else {
- throw new SolrException(ErrorCode.BAD_REQUEST, "Unexpected number of replicas, replicationFactor, " +
- Replica.Type.NRT + " or " + Replica.Type.TLOG + " must be greater than 0");
- }
-
- // Get the first node matching the shard to restore in
- String node;
- for (ReplicaPosition replicaPosition : replicaPositions) {
- if (Objects.equals(replicaPosition.shard, slice.getName())) {
- node = replicaPosition.node;
- propMap.put(CoreAdminParams.NODE, node);
- replicaPositions.remove(replicaPosition);
- break;
+ //Create one replica per shard and copy backed up data to it
+ for (Slice slice : restoreCollection.getSlices()) {
+ String sliceName = slice.getName();
+ log.info("Adding replica for shard={} collection={} ", sliceName, restoreCollection);
+ HashMap<String, Object> propMap = new HashMap<>();
+ propMap.put(Overseer.QUEUE_OPERATION, CREATESHARD);
+ propMap.put(COLLECTION_PROP, restoreCollection.getName());
+ propMap.put(SHARD_ID_PROP, sliceName);
+
+ if (numNrtReplicas >= 1) {
+ propMap.put(REPLICA_TYPE, Replica.Type.NRT.name());
+ } else if (numTlogReplicas >= 1) {
+ propMap.put(REPLICA_TYPE, Replica.Type.TLOG.name());
+ } else {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Unexpected number of replicas, replicationFactor, " +
+ Replica.Type.NRT + " or " + Replica.Type.TLOG + " must be greater than 0");
}
- }
- // add async param
- if (asyncId != null) {
- propMap.put(ASYNC, asyncId);
- }
- ocmh.addPropertyParams(message, propMap);
- final NamedList addReplicaResult = new NamedList();
- ocmh.addReplica(clusterState, new ZkNodeProps(propMap), addReplicaResult, () -> {
- Object addResultFailure = addReplicaResult.get("failure");
- if (addResultFailure != null) {
- SimpleOrderedMap failure = (SimpleOrderedMap) results.get("failure");
- if (failure == null) {
- failure = new SimpleOrderedMap();
- results.add("failure", failure);
+ // Get the first node matching the shard to restore in
+ String node;
+ for (ReplicaPosition replicaPosition : replicaPositions) {
+ if (Objects.equals(replicaPosition.shard, sliceName)) {
+ node = replicaPosition.node;
+ propMap.put(CoreAdminParams.NODE, node);
+ replicaPositions.remove(replicaPosition);
+ break;
}
- failure.addAll((NamedList) addResultFailure);
- } else {
- SimpleOrderedMap success = (SimpleOrderedMap) results.get("success");
- if (success == null) {
- success = new SimpleOrderedMap();
- results.add("success", success);
- }
- success.addAll((NamedList) addReplicaResult.get("success"));
}
- countDownLatch.countDown();
- });
- }
- boolean allIsDone = countDownLatch.await(1, TimeUnit.HOURS);
- if (!allIsDone) {
- throw new TimeoutException("Initial replicas were not created within 1 hour. Timing out.");
- }
- Object failures = results.get("failure");
- if (failures != null && ((SimpleOrderedMap) failures).size() > 0) {
- log.error("Restore failed to create initial replicas.");
- ocmh.cleanupCollection(restoreCollectionName, new NamedList<Object>());
- return;
- }
-
- //refresh the location copy of collection state
- restoreCollection = zkStateReader.getClusterState().getCollection(restoreCollectionName);
+ // add async param
+ if (asyncId != null) {
+ propMap.put(ASYNC, asyncId);
+ }
+ ocmh.addPropertyParams(message, propMap);
+ final NamedList addReplicaResult = new NamedList();
+ ocmh.addReplica(clusterState, new ZkNodeProps(propMap), addReplicaResult, () -> {
+ Object addResultFailure = addReplicaResult.get("failure");
+ if (addResultFailure != null) {
+ SimpleOrderedMap failure = (SimpleOrderedMap) results.get("failure");
+ if (failure == null) {
+ failure = new SimpleOrderedMap();
+ results.add("failure", failure);
+ }
+ failure.addAll((NamedList) addResultFailure);
+ } else {
+ SimpleOrderedMap success = (SimpleOrderedMap) results.get("success");
+ if (success == null) {
+ success = new SimpleOrderedMap();
+ results.add("success", success);
+ }
+ success.addAll((NamedList) addReplicaResult.get("success"));
+ }
+ countDownLatch.countDown();
+ });
+ }
- {
- ShardRequestTracker shardRequestTracker = ocmh.asyncRequestTracker(asyncId);
- // Copy data from backed up index to each replica
- for (Slice slice : restoreCollection.getSlices()) {
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.RESTORECORE.toString());
- params.set(NAME, "snapshot." + slice.getName());
- params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString());
- params.set(CoreAdminParams.BACKUP_REPOSITORY, repo);
- shardRequestTracker.sliceCmd(clusterState, params, null, slice, shardHandler);
+ boolean allIsDone = countDownLatch.await(1, TimeUnit.HOURS);
+ if (!allIsDone) {
+ throw new TimeoutException("Initial replicas were not created within 1 hour. Timing out.");
}
- shardRequestTracker.processResponses(new NamedList(), shardHandler, true, "Could not restore core");
}
- {
+ private void requestReplicasToApplyBufferUpdates(DocCollection restoreCollection, String asyncId, ShardHandler shardHandler) {
ShardRequestTracker shardRequestTracker = ocmh.asyncRequestTracker(asyncId);
for (Slice s : restoreCollection.getSlices()) {
@@ -330,10 +439,7 @@ public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd {
String coreNodeName = r.getCoreName();
Replica.State stateRep = r.getState();
- if (log.isDebugEnabled()) {
- log.debug("Calling REQUESTAPPLYUPDATES on: nodeName={}, coreNodeName={}, state={}", nodeName, coreNodeName,
- stateRep.name());
- }
+ log.debug("Calling REQUESTAPPLYUPDATES on: nodeName={}, coreNodeName={}, state={}", nodeName, coreNodeName, stateRep);
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.REQUESTAPPLYUPDATES.toString());
@@ -342,94 +448,97 @@ public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd {
shardRequestTracker.sendShardRequest(nodeName, params, shardHandler);
}
- shardRequestTracker.processResponses(new NamedList(), shardHandler, true,
+ shardRequestTracker.processResponses(new NamedList<>(), shardHandler, true,
"REQUESTAPPLYUPDATES calls did not succeed");
}
}
//Mark all shards in ACTIVE STATE
- {
+ private void markAllShardsAsActive(DocCollection restoreCollection) throws KeeperException, InterruptedException {
HashMap<String, Object> propMap = new HashMap<>();
propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower());
- propMap.put(ZkStateReader.COLLECTION_PROP, restoreCollectionName);
+ propMap.put(ZkStateReader.COLLECTION_PROP, restoreCollection.getName());
for (Slice shard : restoreCollection.getSlices()) {
propMap.put(shard.getName(), Slice.State.ACTIVE.toString());
}
ocmh.overseer.offerStateUpdate((Utils.toJSON(new ZkNodeProps(propMap))));
}
- if (totalReplicasPerShard > 1) {
- if (log.isInfoEnabled()) {
- log.info("Adding replicas to restored collection={}", restoreCollection.getName());
- }
- for (Slice slice : restoreCollection.getSlices()) {
-
- //Add the remaining replicas for each shard, considering it's type
- int createdNrtReplicas = 0, createdTlogReplicas = 0, createdPullReplicas = 0;
-
- // We already created either a NRT or an TLOG replica as leader
- if (numNrtReplicas > 0) {
- createdNrtReplicas++;
- } else if (createdTlogReplicas > 0) {
- createdTlogReplicas++;
+ private void addReplicasToShards(@SuppressWarnings({"rawtypes"}) NamedList results,
+ ClusterState clusterState,
+ DocCollection restoreCollection,
+ List<ReplicaPosition> replicaPositions,
+ String asyncId) throws Exception {
+ int totalReplicasPerShard = numNrtReplicas + numTlogReplicas + numPullReplicas;
+ if (totalReplicasPerShard > 1) {
+ if (log.isInfoEnabled()) {
+ log.info("Adding replicas to restored collection={}", restoreCollection.getName());
}
+ for (Slice slice : restoreCollection.getSlices()) {
+
+ //Add the remaining replicas for each shard, considering it's type
+ int createdNrtReplicas = 0, createdTlogReplicas = 0, createdPullReplicas = 0;
- for (int i = 1; i < totalReplicasPerShard; i++) {
- Replica.Type typeToCreate;
- if (createdNrtReplicas < numNrtReplicas) {
+ // We already created either a NRT or an TLOG replica as leader
+ if (numNrtReplicas > 0) {
createdNrtReplicas++;
- typeToCreate = Replica.Type.NRT;
- } else if (createdTlogReplicas < numTlogReplicas) {
+ } else if (numTlogReplicas > 0) {
createdTlogReplicas++;
- typeToCreate = Replica.Type.TLOG;
- } else {
- createdPullReplicas++;
- typeToCreate = Replica.Type.PULL;
- assert createdPullReplicas <= numPullReplicas : "Unexpected number of replicas";
}
- if (log.isDebugEnabled()) {
- log.debug("Adding replica for shard={} collection={} of type {} ", slice.getName(), restoreCollection, typeToCreate);
- }
- HashMap<String, Object> propMap = new HashMap<>();
- propMap.put(COLLECTION_PROP, restoreCollectionName);
- propMap.put(SHARD_ID_PROP, slice.getName());
- propMap.put(REPLICA_TYPE, typeToCreate.name());
-
- // Get the first node matching the shard to restore in
- String node;
- for (ReplicaPosition replicaPosition : replicaPositions) {
- if (Objects.equals(replicaPosition.shard, slice.getName())) {
- node = replicaPosition.node;
- propMap.put(CoreAdminParams.NODE, node);
- replicaPositions.remove(replicaPosition);
- break;
+ for (int i = 1; i < totalReplicasPerShard; i++) {
+ Replica.Type typeToCreate;
+ if (createdNrtReplicas < numNrtReplicas) {
+ createdNrtReplicas++;
+ typeToCreate = Replica.Type.NRT;
+ } else if (createdTlogReplicas < numTlogReplicas) {
+ createdTlogReplicas++;
+ typeToCreate = Replica.Type.TLOG;
+ } else {
+ createdPullReplicas++;
+ typeToCreate = Replica.Type.PULL;
+ assert createdPullReplicas <= numPullReplicas : "Unexpected number of replicas";
}
- }
- // add async param
- if (asyncId != null) {
- propMap.put(ASYNC, asyncId);
- }
- ocmh.addPropertyParams(message, propMap);
+ if (log.isDebugEnabled()) {
+ log.debug("Adding replica for shard={} collection={} of type {} ", slice.getName(), restoreCollection, typeToCreate);
+ }
+ HashMap<String, Object> propMap = new HashMap<>();
+ propMap.put(COLLECTION_PROP, restoreCollection.getName());
+ propMap.put(SHARD_ID_PROP, slice.getName());
+ propMap.put(REPLICA_TYPE, typeToCreate.name());
+
+ // Get the first node matching the shard to restore in
+ String node;
+ for (ReplicaPosition replicaPosition : replicaPositions) {
+ if (Objects.equals(replicaPosition.shard, slice.getName())) {
+ node = replicaPosition.node;
+ propMap.put(CoreAdminParams.NODE, node);
+ replicaPositions.remove(replicaPosition);
+ break;
+ }
+ }
- ocmh.addReplica(zkStateReader.getClusterState(), new ZkNodeProps(propMap), results, null);
+ // add async param
+ if (asyncId != null) {
+ propMap.put(ASYNC, asyncId);
+ }
+ ocmh.addPropertyParams(message, propMap);
+
+ ocmh.addReplica(clusterState, new ZkNodeProps(propMap), results, null);
+ }
}
}
}
- if (backupCollectionAlias != null && !backupCollectionAlias.equals(backupCollection)) {
- log.debug("Restoring alias {} -> {}", backupCollectionAlias, backupCollection);
- ocmh.zkStateReader.aliasesManager
- .applyModificationAndExportToZk(a -> a.cloneWithCollectionAlias(backupCollectionAlias, backupCollection));
+ private void restoringAlias(BackupProperties properties) {
+ String backupCollection = properties.getCollection();
+ String backupCollectionAlias = properties.getCollectionAlias();
+ if (backupCollectionAlias != null && !backupCollectionAlias.equals(backupCollection)) {
+ log.debug("Restoring alias {} -> {}", backupCollectionAlias, backupCollection);
+ ocmh.zkStateReader.aliasesManager
+ .applyModificationAndExportToZk(a -> a.cloneWithCollectionAlias(backupCollectionAlias, backupCollection));
+ }
}
-
- log.info("Completed restoring collection={} backupName={}", restoreCollection, backupName);
-
- }
-
- private int getInt(ZkNodeProps message, String propertyName, Integer count, int defaultValue) {
- Integer value = message.getInt(propertyName, count);
- return value!=null ? value:defaultValue;
}
}
diff --git a/solr/core/src/java/org/apache/solr/core/backup/AggregateBackupStats.java b/solr/core/src/java/org/apache/solr/core/backup/AggregateBackupStats.java
new file mode 100644
index 00000000000..1a0c05a3189
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/core/backup/AggregateBackupStats.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.core.backup;
+
+/**
+ * Aggregate stats from multiple {@link ShardBackupMetadata}
+ *
+ * Counted stats may represent multiple shards within a given {@link BackupId}, or span multiple different {@link BackupId BackupIds}.
+ */
+public class AggregateBackupStats {
+ private int numFiles = 0;
+ private long totalSize = 0;
+
+ public AggregateBackupStats() {
+ }
+
+ public void add(ShardBackupMetadata shardBackupMetadata) {
+ numFiles += shardBackupMetadata.numFiles();
+ totalSize += shardBackupMetadata.totalSize();
+ }
+
+ public int getNumFiles() {
+ return numFiles;
+ }
+
+ public long getTotalSize() {
+ return totalSize;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupFilePaths.java b/solr/core/src/java/org/apache/solr/core/backup/BackupFilePaths.java
new file mode 100644
index 00000000000..93452ee6bed
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/core/backup/BackupFilePaths.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.core.backup;
+
+import org.apache.solr.core.backup.repository.BackupRepository;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.apache.solr.core.backup.BackupId.TRADITIONAL_BACKUP;
+import static org.apache.solr.core.backup.BackupManager.ZK_STATE_DIR;
+
+/**
+ * Utility class for getting paths related to backups, or parsing information out of those paths.
+ */
+public class BackupFilePaths {
+
+ private static final Pattern BACKUP_PROPS_ID_PTN = Pattern.compile("backup_([0-9]+).properties");
+ private BackupRepository repository;
+ private URI backupLoc;
+
+ /**
+ * Create a BackupFilePaths object.
+ *
+ * @param repository the repository; used primarily to resolve URIs.
+ * @param backupLoc the root location for a named backup. For traditional backups this is expected to take the form
+ * baseLocation/backupName. For incremental backups this is expected to be of the form
+ * baseLocation/backupName/collectionName.
+ */
+ public BackupFilePaths(BackupRepository repository, URI backupLoc) {
+ this.repository = repository;
+ this.backupLoc = backupLoc;
+ }
+
+ /**
+ * Return a URI for the 'index' location, responsible for holding index files for all backups at this location.
+ *
+ * Only valid for incremental backups.
+ */
+ public URI getIndexDir() {
+ return repository.resolve(backupLoc, "index");
+ }
+
+ /**
+ * Return a URI for the 'shard_backup_metadata' location, which contains metadata files about each shard backup.
+ *
+ * Only valid for incremental backups.
+ */
+ public URI getShardBackupMetadataDir() {
+ return repository.resolve(backupLoc, "shard_backup_metadata");
+ }
+
+ public URI getBackupLocation() {
+ return backupLoc;
+ }
+
+ /**
+ * Create all locations required to store an incremental backup.
+ *
+ * @throws IOException for issues encountered using repository to create directories
+ */
+ public void createIncrementalBackupFolders() throws IOException {
+ if (!repository.exists(backupLoc)) {
+ repository.createDirectory(backupLoc);
+ }
+ URI indexDir = getIndexDir();
+ if (!repository.exists(indexDir)) {
+ repository.createDirectory(indexDir);
+ }
+
+ URI shardBackupMetadataDir = getShardBackupMetadataDir();
+ if (!repository.exists(shardBackupMetadataDir)) {
+ repository.createDirectory(shardBackupMetadataDir);
+ }
+ }
+
+ /**
+ * Get the directory name used to hold backed up ZK state
+ *
+ * Valid for both incremental and traditional backups.
+ *
+ * @param id the ID of the backup in question
+ */
+ public static String getZkStateDir(BackupId id) {
+ if (id.id == TRADITIONAL_BACKUP) {
+ return ZK_STATE_DIR;
+ }
+ return String.format(Locale.ROOT, "%s_%d/", ZK_STATE_DIR, id.id);
+ }
+
+ /**
+ * Get the filename of the top-level backup properties file
+ *
+ * Valid for both incremental and traditional backups.
+ *
+ * @param id the ID of the backup in question
+ */
+ public static String getBackupPropsName(BackupId id) {
+ if (id.id == TRADITIONAL_BACKUP) {
+ return BackupManager.TRADITIONAL_BACKUP_PROPS_FILE;
+ }
+ return getBackupPropsName(id.id);
+ }
+
+ /**
+ * Identify all strings which appear to be the filename of a top-level backup properties file.
+ *
+ * Only valid for incremental backups.
+ *
+ * @param listFiles a list of strings, filenames which may or may not correspond to backup properties files
+ */
+ public static List<BackupId> findAllBackupIdsFromFileListing(String[] listFiles) {
+ List<BackupId> result = new ArrayList<>();
+ for (String file: listFiles) {
+ Matcher m = BACKUP_PROPS_ID_PTN.matcher(file);
+ if (m.find()) {
+ result.add(new BackupId(Integer.parseInt(m.group(1))));
+ }
+ }
+
+ return result;
+ }
+
+ /**
+ * Identify the string from an array of filenames which represents the most recent top-level backup properties file.
+ *
+ * Only valid for incremental backups.
+ *
+ * @param listFiles a list of strings, filenames which may or may not correspond to backup properties files.
+ */
+ public static Optional<BackupId> findMostRecentBackupIdFromFileListing(String[] listFiles) {
+ return findAllBackupIdsFromFileListing(listFiles).stream().max(Comparator.comparingInt(o -> o.id));
+ }
+
+ /**
+ * Builds the URI for the backup location given the user-provided 'location' and backup 'name'.
+ *
+ * @param repository the backup repository, used to list files and resolve URI's.
+ * @param location a URI representing the repository location holding each backup name
+ * @param backupName the specific backup name to create a URI for
+ */
+ public static URI buildExistingBackupLocationURI(BackupRepository repository, URI location, String backupName) throws IOException {
+ final URI backupNameUri = repository.resolve(location, backupName);
+ final String[] entries = repository.listAll(backupNameUri);
+ final boolean incremental = ! Arrays.stream(entries).anyMatch(entry -> entry.equals(BackupManager.TRADITIONAL_BACKUP_PROPS_FILE));
+ if (incremental) {
+ // Incremental backups have an additional URI path component representing the collection that was backed up.
+ // This collection directory is the path assumed by other backup code.
+ if (entries.length != 1) {
+ throw new IllegalStateException("Incremental backup URI [" + backupNameUri + "] expected to hold a single directory");
+ }
+ final String collectionName = entries[0];
+ return repository.resolve(backupNameUri, entries[0]);
+ } else {
+ return backupNameUri;
+ }
+ }
+
+ private static String getBackupPropsName(int id) {
+ return String.format(Locale.ROOT, "backup_%d.properties", id);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupId.java b/solr/core/src/java/org/apache/solr/core/backup/BackupId.java
new file mode 100644
index 00000000000..290564616db
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/core/backup/BackupId.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.core.backup;
+
+/**
+ * Represents the ID of a particular backup.
+ *
+ * Backup IDs are used to track different backup points stored at the same backup-location under the same backup-name.
+ *
+ * Incremental backups can have any non-negative integer as an ID, and ID's are expected to increase sequentially.
+ *
+ * Traditional (now-deprecated) 'full-snapshot' backups only support a single backup point per name per location. So
+ * these all have the same ID value of {@link #TRADITIONAL_BACKUP}
+ */
+public class BackupId implements Comparable<BackupId>{
+ public static final int TRADITIONAL_BACKUP = -1;
+
+ public final int id;
+
+ public BackupId(int id) {
+ this.id = id;
+ }
+
+ public static BackupId zero() {
+ return new BackupId(0);
+ }
+
+ public static BackupId traditionalBackup() {
+ return new BackupId(TRADITIONAL_BACKUP);
+ }
+
+ public BackupId nextBackupId() {
+ return new BackupId(id+1);
+ }
+
+ public int getId() {
+ return id;
+ }
+
+ @Override
+ public int compareTo(BackupId o) {
+ return Integer.compare(this.id, o.id);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ BackupId backupId = (BackupId) o;
+ return id == backupId.id;
+ }
+
+ @Override
+ public int hashCode() {
+ return id;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
index deae3604c79..c5b48c77739 100644
--- a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
+++ b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
@@ -17,10 +17,8 @@
package org.apache.solr.core.backup;
import java.io.IOException;
-import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
-import java.io.Reader;
import java.io.Writer;
import java.lang.invoke.MethodHandles;
import java.net.URI;
@@ -28,7 +26,7 @@ import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
-import java.util.Properties;
+import java.util.Optional;
import com.google.common.base.Preconditions;
import org.apache.lucene.store.IOContext;
@@ -43,7 +41,6 @@ import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.backup.repository.BackupRepository;
import org.apache.solr.core.backup.repository.BackupRepository.PathType;
-import org.apache.solr.util.PropertiesInputStream;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
@@ -56,7 +53,7 @@ import org.slf4j.LoggerFactory;
public class BackupManager {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String COLLECTION_PROPS_FILE = "collection_state.json";
- public static final String BACKUP_PROPS_FILE = "backup.properties";
+ public static final String TRADITIONAL_BACKUP_PROPS_FILE = "backup.properties";
public static final String ZK_STATE_DIR = "zk_backup";
public static final String CONFIG_STATE_DIR = "configs";
@@ -64,15 +61,90 @@ public class BackupManager {
public static final String COLLECTION_NAME_PROP = "collection";
public static final String COLLECTION_ALIAS_PROP = "collectionAlias";
public static final String BACKUP_NAME_PROP = "backupName";
- public static final String INDEX_VERSION_PROP = "index.version";
+ public static final String INDEX_VERSION_PROP = "indexVersion";
public static final String START_TIME_PROP = "startTime";
protected final ZkStateReader zkStateReader;
protected final BackupRepository repository;
+ protected final BackupId backupId;
+ protected final URI backupPath;
+ protected final String existingPropsFile;
- public BackupManager(BackupRepository repository, ZkStateReader zkStateReader) {
+
+ private BackupManager(BackupRepository repository,
+ URI backupPath,
+ ZkStateReader zkStateReader,
+ String existingPropsFile,
+ BackupId backupId) {
this.repository = Objects.requireNonNull(repository);
+ this.backupPath = backupPath;
this.zkStateReader = Objects.requireNonNull(zkStateReader);
+ this.existingPropsFile = existingPropsFile;
+ this.backupId = backupId;
+ }
+
+ public static BackupManager forIncrementalBackup(BackupRepository repository,
+ ZkStateReader stateReader,
+ URI backupPath) {
+ Objects.requireNonNull(repository);
+ Objects.requireNonNull(stateReader);
+
+ Optional<BackupId> lastBackupId = BackupFilePaths.findMostRecentBackupIdFromFileListing(repository.listAllOrEmpty(backupPath));
+
+ return new BackupManager(repository, backupPath, stateReader, lastBackupId
+ .map(id ->BackupFilePaths.getBackupPropsName(id)).orElse(null),
+ lastBackupId.map(BackupId::nextBackupId).orElse(BackupId.zero()));
+ }
+
+ public static BackupManager forBackup(BackupRepository repository,
+ ZkStateReader stateReader,
+ URI backupPath) {
+ Objects.requireNonNull(repository);
+ Objects.requireNonNull(stateReader);
+
+ return new BackupManager(repository, backupPath, stateReader, null, BackupId.traditionalBackup());
+ }
+
+ public static BackupManager forRestore(BackupRepository repository,
+ ZkStateReader stateReader,
+ URI backupPath,
+ int bid) throws IOException {
+ Objects.requireNonNull(repository);
+ Objects.requireNonNull(stateReader);
+
+ BackupId backupId = new BackupId(bid);
+ String backupPropsName = BackupFilePaths.getBackupPropsName(backupId);
+ if (!repository.exists(repository.resolve(backupPath, backupPropsName))) {
+ throw new IllegalStateException("Backup id " + bid + " was not found");
+ }
+
+ return new BackupManager(repository, backupPath, stateReader, backupPropsName, backupId);
+ }
+
+ public static BackupManager forRestore(BackupRepository repository,
+ ZkStateReader stateReader,
+ URI backupPath) throws IOException {
+ Objects.requireNonNull(repository);
+ Objects.requireNonNull(stateReader);
+
+ if (!repository.exists(backupPath)) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Couldn't restore since doesn't exist: " + backupPath);
+ }
+
+ Optional<BackupId> opFileGen = BackupFilePaths.findMostRecentBackupIdFromFileListing(repository.listAll(backupPath));
+ if (opFileGen.isPresent()) {
+ BackupId backupPropFile = opFileGen.get();
+ return new BackupManager(repository, backupPath, stateReader, BackupFilePaths.getBackupPropsName(backupPropFile),
+ backupPropFile);
+ } else if (repository.exists(repository.resolve(backupPath, TRADITIONAL_BACKUP_PROPS_FILE))){
+ return new BackupManager(repository, backupPath, stateReader, TRADITIONAL_BACKUP_PROPS_FILE, null);
+ } else {
+ throw new IllegalStateException("No " + TRADITIONAL_BACKUP_PROPS_FILE + " was found, the backup does not exist or not complete");
+ }
+ }
+
+ public final BackupId getBackupId() {
+ return backupId;
}
/**
@@ -85,57 +157,49 @@ public class BackupManager {
/**
* This method returns the configuration parameters for the specified backup.
*
- * @param backupLoc The base path used to store the backup data.
- * @param backupId The unique name for the backup whose configuration params are required.
* @return the configuration parameters for the specified backup.
* @throws IOException In case of errors.
*/
- public Properties readBackupProperties(URI backupLoc, String backupId) throws IOException {
- Objects.requireNonNull(backupLoc);
- Objects.requireNonNull(backupId);
-
- // Backup location
- URI backupPath = repository.resolve(backupLoc, backupId);
- if (!repository.exists(backupPath)) {
- throw new SolrException(ErrorCode.SERVER_ERROR, "Couldn't restore since doesn't exist: " + backupPath);
+ public BackupProperties readBackupProperties() throws IOException {
+ if (existingPropsFile == null) {
+ throw new IllegalStateException("No " + TRADITIONAL_BACKUP_PROPS_FILE + " was found, the backup does not exist or not complete");
}
- Properties props = new Properties();
- try (Reader is = new InputStreamReader(new PropertiesInputStream(
- repository.openInput(backupPath, BACKUP_PROPS_FILE, IOContext.DEFAULT)), StandardCharsets.UTF_8)) {
- props.load(is);
- return props;
+ return BackupProperties.readFrom(repository, backupPath, existingPropsFile);
+ }
+
+ public Optional<BackupProperties> tryReadBackupProperties() throws IOException {
+ if (existingPropsFile != null) {
+ return Optional.of(BackupProperties.readFrom(repository, backupPath, existingPropsFile));
}
+
+ return Optional.empty();
}
/**
* This method stores the backup properties at the specified location in the repository.
*
- * @param backupLoc The base path used to store the backup data.
- * @param backupId The unique name for the backup whose configuration params are required.
* @param props The backup properties
* @throws IOException in case of I/O error
*/
- public void writeBackupProperties(URI backupLoc, String backupId, Properties props) throws IOException {
- URI dest = repository.resolve(backupLoc, backupId, BACKUP_PROPS_FILE);
+ public void writeBackupProperties(BackupProperties props) throws IOException {
+ URI dest = repository.resolve(backupPath, BackupFilePaths.getBackupPropsName(backupId));
try (Writer propsWriter = new OutputStreamWriter(repository.createOutput(dest), StandardCharsets.UTF_8)) {
- props.store(propsWriter, "Backup properties file");
+ props.store(propsWriter);
}
}
/**
* This method reads the meta-data information for the backed-up collection.
*
- * @param backupLoc The base path used to store the backup data.
- * @param backupId The unique name for the backup.
* @param collectionName The name of the collection whose meta-data is to be returned.
* @return the meta-data information for the backed-up collection.
* @throws IOException in case of errors.
*/
- public DocCollection readCollectionState(URI backupLoc, String backupId, String collectionName) throws IOException {
+ public DocCollection readCollectionState(String collectionName) throws IOException {
Objects.requireNonNull(collectionName);
- URI zkStateDir = repository.resolve(backupLoc, backupId, ZK_STATE_DIR);
+ URI zkStateDir = getZkStateDir();
try (IndexInput is = repository.openInput(zkStateDir, COLLECTION_PROPS_FILE, IOContext.DEFAULT)) {
byte[] arr = new byte[(int) is.length()]; // probably ok since the json file should be small.
is.readBytes(arr, 0, (int) is.length());
@@ -147,15 +211,13 @@ public class BackupManager {
/**
* This method writes the collection meta-data to the specified location in the repository.
*
- * @param backupLoc The base path used to store the backup data.
- * @param backupId The unique name for the backup.
* @param collectionName The name of the collection whose meta-data is being stored.
* @param collectionState The collection meta-data to be stored.
* @throws IOException in case of I/O errors.
*/
- public void writeCollectionState(URI backupLoc, String backupId, String collectionName,
+ public void writeCollectionState(String collectionName,
DocCollection collectionState) throws IOException {
- URI dest = repository.resolve(backupLoc, backupId, ZK_STATE_DIR, COLLECTION_PROPS_FILE);
+ URI dest = getZkStateDir(COLLECTION_PROPS_FILE);
try (OutputStream collectionStateOs = repository.createOutput(dest)) {
collectionStateOs.write(Utils.toJSON(Collections.singletonMap(collectionName, collectionState)));
}
@@ -164,38 +226,38 @@ public class BackupManager {
/**
* This method uploads the Solr configuration files to the desired location in Zookeeper.
*
- * @param backupLoc The base path used to store the backup data.
- * @param backupId The unique name for the backup.
* @param sourceConfigName The name of the config to be copied
* @param targetConfigName The name of the config to be created.
* @throws IOException in case of I/O errors.
*/
- public void uploadConfigDir(URI backupLoc, String backupId, String sourceConfigName, String targetConfigName)
- throws IOException {
- URI source = repository.resolve(backupLoc, backupId, ZK_STATE_DIR, CONFIG_STATE_DIR, sourceConfigName);
+ public void uploadConfigDir(String sourceConfigName, String targetConfigName)
+ throws IOException {
+ URI source = getZkStateDir(CONFIG_STATE_DIR, sourceConfigName);
String zkPath = ZkConfigManager.CONFIGS_ZKNODE + "/" + targetConfigName;
+
+ Preconditions.checkState(repository.exists(source), "Path {} does not exist", source);
+ Preconditions.checkState(repository.getPathType(source) == PathType.DIRECTORY,
+ "Path {} is not a directory", zkPath);
uploadToZk(zkStateReader.getZkClient(), source, zkPath);
}
/**
* This method stores the contents of a specified Solr config at the specified location in repository.
*
- * @param backupLoc The base path used to store the backup data.
- * @param backupId The unique name for the backup.
* @param configName The name of the config to be saved.
* @throws IOException in case of I/O errors.
*/
- public void downloadConfigDir(URI backupLoc, String backupId, String configName) throws IOException {
- URI dest = repository.resolve(backupLoc, backupId, ZK_STATE_DIR, CONFIG_STATE_DIR, configName);
- repository.createDirectory(repository.resolve(backupLoc, backupId, ZK_STATE_DIR));
- repository.createDirectory(repository.resolve(backupLoc, backupId, ZK_STATE_DIR, CONFIG_STATE_DIR));
+ public void downloadConfigDir(String configName) throws IOException {
+ URI dest = getZkStateDir(CONFIG_STATE_DIR, configName);
+ repository.createDirectory(getZkStateDir());
+ repository.createDirectory(getZkStateDir(CONFIG_STATE_DIR));
repository.createDirectory(dest);
downloadFromZK(zkStateReader.getZkClient(), ZkConfigManager.CONFIGS_ZKNODE + "/" + configName, dest);
}
- public void uploadCollectionProperties(URI backupLoc, String backupId, String collectionName) throws IOException {
- URI sourceDir = repository.resolve(backupLoc, backupId, ZK_STATE_DIR);
+ public void uploadCollectionProperties(String collectionName) throws IOException {
+ URI sourceDir = getZkStateDir();
URI source = repository.resolve(sourceDir, ZkStateReader.COLLECTION_PROPS_ZKNODE);
if (!repository.exists(source)) {
// No collection properties to restore
@@ -209,12 +271,12 @@ public class BackupManager {
zkStateReader.getZkClient().create(zkPath, arr, CreateMode.PERSISTENT, true);
} catch (KeeperException | InterruptedException e) {
throw new IOException("Error uploading file to zookeeper path " + source.toString() + " to " + zkPath,
- SolrZkClient.checkInterrupted(e));
+ SolrZkClient.checkInterrupted(e));
}
}
- public void downloadCollectionProperties(URI backupLoc, String backupId, String collectionName) throws IOException {
- URI dest = repository.resolve(backupLoc, backupId, ZK_STATE_DIR, ZkStateReader.COLLECTION_PROPS_ZKNODE);
+ public void downloadCollectionProperties(String collectionName) throws IOException {
+ URI dest = getZkStateDir(ZkStateReader.COLLECTION_PROPS_ZKNODE);
String zkPath = ZkStateReader.COLLECTIONS_ZKNODE + '/' + collectionName + '/' + ZkStateReader.COLLECTION_PROPS_ZKNODE;
@@ -230,15 +292,12 @@ public class BackupManager {
}
} catch (KeeperException | InterruptedException e) {
throw new IOException("Error downloading file from zookeeper path " + zkPath + " to " + dest.toString(),
- SolrZkClient.checkInterrupted(e));
+ SolrZkClient.checkInterrupted(e));
}
}
private void downloadFromZK(SolrZkClient zkClient, String zkPath, URI dir) throws IOException {
try {
- if (!repository.exists(dir)) {
- repository.createDirectory(dir);
- }
List<String> files = zkClient.getChildren(zkPath, null, true);
for (String file : files) {
List<String> children = zkClient.getChildren(zkPath + "/" + file, null, true);
@@ -249,20 +308,20 @@ public class BackupManager {
os.write(data);
}
} else {
+ URI uri = repository.resolve(dir, file);
+ if (!repository.exists(uri)) {
+ repository.createDirectory(uri);
+ }
downloadFromZK(zkClient, zkPath + "/" + file, repository.resolve(dir, file));
}
}
} catch (KeeperException | InterruptedException e) {
throw new IOException("Error downloading files from zookeeper path " + zkPath + " to " + dir.toString(),
- SolrZkClient.checkInterrupted(e));
+ SolrZkClient.checkInterrupted(e));
}
}
private void uploadToZk(SolrZkClient zkClient, URI sourceDir, String destZkPath) throws IOException {
- Preconditions.checkArgument(repository.exists(sourceDir), "Path {} does not exist", sourceDir);
- Preconditions.checkArgument(repository.getPathType(sourceDir) == PathType.DIRECTORY,
- "Path {} is not a directory", sourceDir);
-
for (String file : repository.listAll(sourceDir)) {
String zkNodePath = destZkPath + "/" + file;
URI path = repository.resolve(sourceDir, file);
@@ -290,4 +349,19 @@ public class BackupManager {
}
}
}
+
+ private URI getZkStateDir(String... subFolders) {
+ URI zkStateDir;
+ if (backupId != null) {
+ final String zkBackupFolder = BackupFilePaths.getZkStateDir(backupId);
+ zkStateDir = repository.resolve(backupPath, zkBackupFolder);
+ } else {
+ zkStateDir = repository.resolve(backupPath, ZK_STATE_DIR);
+ }
+
+ if (subFolders.length == 0) {
+ return zkStateDir;
+ }
+ return repository.resolve(zkStateDir, subFolders);
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupProperties.java b/solr/core/src/java/org/apache/solr/core/backup/BackupProperties.java
new file mode 100644
index 00000000000..94d66275c79
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/core/backup/BackupProperties.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.core.backup;
+
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.Version;
+import org.apache.solr.common.params.CollectionAdminParams;
+import org.apache.solr.core.backup.repository.BackupRepository;
+import org.apache.solr.util.PropertiesInputStream;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.Writer;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.stream.Collectors;
+
+/**
+ * Represents a backup[-*].properties file and the methods to read/write it.
+ *
+ * These files live in a different location and hold different metadata depending on the backup format used. The (now
+ * deprecated) traditional 'full-snapshot' backup format places this file at $LOCATION/$NAME, while the preferred
+ * incremental backup format stores these files in $LOCATION/$NAME/$COLLECTION.
+ */
+public class BackupProperties {
+
+ private double indexSizeMB;
+ private int indexFileCount;
+
+ private Properties properties;
+
+ private BackupProperties(Properties properties) {
+ this.properties = properties;
+ }
+
+ public static BackupProperties create(String backupName,
+ String collectionName,
+ String extCollectionName,
+ String configName) {
+ Properties properties = new Properties();
+ properties.put(BackupManager.BACKUP_NAME_PROP, backupName);
+ properties.put(BackupManager.COLLECTION_NAME_PROP, collectionName);
+ properties.put(BackupManager.COLLECTION_ALIAS_PROP, extCollectionName);
+ properties.put(CollectionAdminParams.COLL_CONF, configName);
+ properties.put(BackupManager.START_TIME_PROP, Instant.now().toString());
+ properties.put(BackupManager.INDEX_VERSION_PROP, Version.LATEST.toString());
+
+ return new BackupProperties(properties);
+ }
+
+ public static Optional<BackupProperties> readFromLatest(BackupRepository repository, URI backupPath) throws IOException {
+ Optional<BackupId> lastBackupId = BackupFilePaths.findMostRecentBackupIdFromFileListing(repository.listAllOrEmpty(backupPath));
+
+ if (!lastBackupId.isPresent()) {
+ return Optional.empty();
+ }
+
+ return Optional.of(readFrom(repository, backupPath, BackupFilePaths.getBackupPropsName(lastBackupId.get())));
+ }
+
+ public static BackupProperties readFrom(BackupRepository repository, URI backupPath, String fileName) throws IOException {
+ Properties props = new Properties();
+ try (Reader is = new InputStreamReader(new PropertiesInputStream(
+ repository.openInput(backupPath, fileName, IOContext.DEFAULT)), StandardCharsets.UTF_8)) {
+ props.load(is);
+ return new BackupProperties(props);
+ }
+ }
+
+ public List<String> getAllShardBackupMetadataFiles() {
+ return properties.entrySet()
+ .stream()
+ .filter(entry -> entry.getKey().toString().endsWith(".md"))
+ .map(entry -> entry.getValue().toString())
+ .collect(Collectors.toList());
+ }
+
+ public void countIndexFiles(int numFiles, double sizeMB) {
+ indexSizeMB += sizeMB;
+ indexFileCount += numFiles;
+ }
+
+ public Optional<ShardBackupId> getShardBackupIdFor(String shardName) {
+ String key = getKeyForShardBackupId(shardName);
+ if (properties.containsKey(key)) {
+ return Optional.of(ShardBackupId.fromShardMetadataFilename(properties.getProperty(key)));
+ }
+ return Optional.empty();
+ }
+
+ public ShardBackupId putAndGetShardBackupIdFor(String shardName, int backupId) {
+ final ShardBackupId shardBackupId = new ShardBackupId(shardName, new BackupId(backupId));
+ properties.put(getKeyForShardBackupId(shardName), shardBackupId.getBackupMetadataFilename());
+ return shardBackupId;
+ }
+
+ private String getKeyForShardBackupId(String shardName) {
+ return shardName+".md";
+ }
+
+
+ public void store(Writer propsWriter) throws IOException {
+ properties.put("indexSizeMB", String.valueOf(indexSizeMB));
+ properties.put("indexFileCount", String.valueOf(indexFileCount));
+ properties.store(propsWriter, "Backup properties file");
+ }
+
+ public String getCollection() {
+ return properties.getProperty(BackupManager.COLLECTION_NAME_PROP);
+ }
+
+
+ public String getCollectionAlias() {
+ return properties.getProperty(BackupManager.COLLECTION_ALIAS_PROP);
+ }
+
+ public String getConfigName() {
+ return properties.getProperty(CollectionAdminParams.COLL_CONF);
+ }
+
+ public String getStartTime() {
+ return properties.getProperty(BackupManager.START_TIME_PROP);
+ }
+
+ public String getIndexVersion() {
+ return properties.getProperty(BackupManager.INDEX_VERSION_PROP);
+ }
+
+ public Map<Object, Object> getDetails() {
+ Map<Object, Object> result = new HashMap<>(properties);
+ result.remove(BackupManager.BACKUP_NAME_PROP);
+ result.remove(BackupManager.COLLECTION_NAME_PROP);
+ result.put("indexSizeMB", Double.valueOf(properties.getProperty("indexSizeMB")));
+ result.put("indexFileCount", Integer.valueOf(properties.getProperty("indexFileCount")));
+
+ Map<String, String> shardBackupIds = new HashMap<>();
+ Iterator<Object> keyIt = result.keySet().iterator();
+ while (keyIt.hasNext()) {
+ String key = keyIt.next().toString();
+ if (key.endsWith(".md")) {
+ shardBackupIds.put(key.substring(0, key.length() - 3), properties.getProperty(key));
+ keyIt.remove();
+ }
+ }
+ result.put("shardBackupIds", shardBackupIds);
+ return result;
+ }
+
+ public String getBackupName() {
+ return properties.getProperty(BackupManager.BACKUP_NAME_PROP);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/core/backup/Checksum.java b/solr/core/src/java/org/apache/solr/core/backup/Checksum.java
new file mode 100644
index 00000000000..6338708b3af
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/core/backup/Checksum.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.core.backup;
+
+import java.util.Objects;
+
+/**
+ * Represents checksum information for an index file being backed up.
+ */
+public class Checksum {
+ public final long checksum;
+ public final long size;
+
+ public Checksum(long checksum, long size) {
+ this.checksum = checksum;
+ this.size = size;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Checksum checksum = (Checksum) o;
+ return size == checksum.size &&
+ this.checksum == checksum.checksum;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(checksum, size);
+ }
+
+}
diff --git a/solr/core/src/java/org/apache/solr/core/backup/ShardBackupId.java b/solr/core/src/java/org/apache/solr/core/backup/ShardBackupId.java
new file mode 100644
index 00000000000..78de78d5f3b
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/core/backup/ShardBackupId.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.core.backup;
+
+/**
+ * Represents the ID of a particular backup point for a particular shard.
+ *
+ * ShardBackupId's only need be unique within a given collection and backup location/name, so in practice they're formed
+ * by combining the shard name with the {@link BackupId} in the form: "md_$SHARDNAME_BACKUPID".
+ *
+ * ShardBackupId's are most often used as a filename to store shard-level metadata for the backup. See
+ * {@link ShardBackupMetadata} for more information.
+ *
+ * @see ShardBackupMetadata
+ */
+public class ShardBackupId {
+ private static final String FILENAME_SUFFIX = ".json";
+ private final String shardName;
+ private final BackupId containingBackupId;
+
+ public ShardBackupId(String shardName, BackupId containingBackupId) {
+ this.shardName = shardName;
+ this.containingBackupId = containingBackupId;
+ }
+
+ public String getShardName() {
+ return shardName;
+ }
+
+ public BackupId getContainingBackupId() {
+ return containingBackupId;
+ }
+
+ public String getIdAsString() {
+ return "md_" + shardName + "_" + containingBackupId.getId();
+ }
+
+ public String getBackupMetadataFilename() {
+ return getIdAsString() + FILENAME_SUFFIX;
+ }
+
+ public static ShardBackupId from(String idString) {
+ final String[] idComponents = idString.split("_");
+ if (idComponents.length != 3) {
+ throw new IllegalArgumentException("Unable to parse invalid ShardBackupId: " + idString);
+ }
+
+ final BackupId containingBackupId = new BackupId(Integer.parseInt(idComponents[2]));
+ return new ShardBackupId(idComponents[1], containingBackupId);
+ }
+
+ public static ShardBackupId fromShardMetadataFilename(String filenameString) {
+ if (! filenameString.endsWith(FILENAME_SUFFIX)) {
+ throw new IllegalArgumentException("'filenameString' arg [" + filenameString + "] does not appear to be a filename");
+ }
+ final String idString = filenameString.substring(0, filenameString.length() - FILENAME_SUFFIX.length());
+ return from(idString);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/core/backup/ShardBackupMetadata.java b/solr/core/src/java/org/apache/solr/core/backup/ShardBackupMetadata.java
new file mode 100644
index 00000000000..4b25950add2
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/core/backup/ShardBackupMetadata.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.core.backup;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.backup.repository.BackupRepository;
+import org.apache.solr.util.PropertiesInputStream;
+
+/**
+ * Represents the shard-backup metadata file.
+ *
+ * The shard-backup metadata file is responsible for holding information about a specific backup-point for a specific
+ * shard. This includes the full list of index files required to restore this shard to the backup-point, with pointers
+ * to where each lives in the repository.
+ *
+ * Shard backup metadata files have names derived from an associated {@link ShardBackupId}, to avoid conflicts between
+ * shards and backupIds.
+ *
+ * Not used by the (now deprecated) traditional 'full-snapshot' backup format.
+ */
+public class ShardBackupMetadata {
+ private Map<String, BackedFile> allFiles = new HashMap<>();
+ private List<String> uniqueFileNames = new ArrayList<>();
+
+ public void addBackedFile(String uniqueFileName, String originalFileName, Checksum fileChecksum) {
+ addBackedFile(new BackedFile(uniqueFileName, originalFileName, fileChecksum));
+ }
+
+ public int numFiles() {
+ return uniqueFileNames.size();
+ }
+
+ public long totalSize() {
+ return allFiles.values().stream().map(bf -> bf.fileChecksum.size).reduce(0L, Long::sum);
+ }
+
+ public void addBackedFile(BackedFile backedFile) {
+ allFiles.put(backedFile.originalFileName, backedFile);
+ uniqueFileNames.add(backedFile.uniqueFileName);
+ }
+
+ public Optional<BackedFile> getFile(String originalFileName) {
+ return Optional.ofNullable(allFiles.get(originalFileName));
+ }
+
+ public List<String> listUniqueFileNames() {
+ return Collections.unmodifiableList(uniqueFileNames);
+ }
+
+ public static ShardBackupMetadata empty() {
+ return new ShardBackupMetadata();
+ }
+
+ public static ShardBackupMetadata from(BackupRepository repository, URI dir, ShardBackupId shardBackupId) throws IOException {
+ final String shardBackupMetadataFilename = shardBackupId.getBackupMetadataFilename();
+ if (!repository.exists(repository.resolve(dir, shardBackupMetadataFilename))) {
+ return null;
+ }
+
+ try (IndexInput is = repository.openInput(dir, shardBackupMetadataFilename, IOContext.DEFAULT)) {
+ return from(new PropertiesInputStream(is));
+ }
+ }
+
+ /**
+ * Storing ShardBackupMetadata at {@code folderURI} with name {@code filename}.
+ * If a file already existed there, overwrite it.
+ */
+ public void store(BackupRepository repository, URI folderURI, ShardBackupId shardBackupId) throws IOException {
+ final String filename = shardBackupId.getBackupMetadataFilename();
+ URI fileURI = repository.resolve(folderURI, filename);
+ if (repository.exists(fileURI)) {
+ repository.delete(folderURI, Collections.singleton(filename), true);
+ }
+
+ try (OutputStream os = repository.createOutput(repository.resolve(folderURI, filename))) {
+ store(os);
+ }
+ }
+
+ public Collection<String> listOriginalFileNames() {
+ return Collections.unmodifiableSet(allFiles.keySet());
+ }
+
+ private void store(OutputStream os) throws IOException {
+ @SuppressWarnings({"rawtypes"})
+ Map<String, Map> map = new HashMap<>();
+
+ for (BackedFile backedFile : allFiles.values()) {
+ Map<String, Object> fileMap = new HashMap<>();
+ fileMap.put("fileName", backedFile.originalFileName);
+ fileMap.put("checksum", backedFile.fileChecksum.checksum);
+ fileMap.put("size", backedFile.fileChecksum.size);
+ map.put(backedFile.uniqueFileName, fileMap);
+ }
+
+ Utils.writeJson(map, os, false);
+ }
+
+ private static ShardBackupMetadata from(InputStream is) {
+ @SuppressWarnings({"unchecked"})
+ Map<String, Object> map = (Map<String, Object>) Utils.fromJSON(is);
+ ShardBackupMetadata shardBackupMetadata = new ShardBackupMetadata();
+ for (String uniqueFileName : map.keySet()) {
+ @SuppressWarnings({"unchecked"})
+ Map<String, Object> fileMap = (Map<String, Object>) map.get(uniqueFileName);
+
+ String fileName = (String) fileMap.get("fileName");
+ long checksum = (long) fileMap.get("checksum");
+ long size = (long) fileMap.get("size");
+ shardBackupMetadata.addBackedFile(new BackedFile(uniqueFileName, fileName, new Checksum(checksum, size)));
+ }
+
+ return shardBackupMetadata;
+ }
+
+ public static class BackedFile {
+ public final String uniqueFileName;
+ public final String originalFileName;
+ public final Checksum fileChecksum;
+
+ BackedFile(String uniqueFileName, String originalFileName, Checksum fileChecksum) {
+ this.uniqueFileName = uniqueFileName;
+ this.originalFileName = originalFileName;
+ this.fileChecksum = fileChecksum;
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepository.java b/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepository.java
index d750f590039..e173ed293ee 100644
--- a/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepository.java
+++ b/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepository.java
@@ -20,11 +20,20 @@ import java.io.Closeable;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
+import java.util.Collection;
+import java.util.Optional;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.IOUtils;
import org.apache.solr.common.params.CoreAdminParams;
+import org.apache.solr.core.DirectoryFactory;
+import org.apache.solr.core.backup.Checksum;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
/**
@@ -47,8 +56,7 @@ public interface BackupRepository extends NamedListInitializedPlugin, Closeable
* Otherwise return the default configuration value for the {@linkplain CoreAdminParams#BACKUP_LOCATION} parameter.
*/
default String getBackupLocation(String override) {
- // If override is null and default backup location is unset, what do we do?
- return override != null ? override : getConfigProperty(CoreAdminParams.BACKUP_LOCATION);
+ return Optional.ofNullable(override).orElse(getConfigProperty(CoreAdminParams.BACKUP_LOCATION));
}
/**
@@ -63,7 +71,7 @@ public interface BackupRepository extends NamedListInitializedPlugin, Closeable
* @param path The path specified by the user.
* @return the URI representation of the user supplied value
*/
- URI createURI(String path);
+ URI createURI(String path);
/**
* This method resolves a URI using the specified path components (as method arguments).
@@ -136,6 +144,7 @@ public interface BackupRepository extends NamedListInitializedPlugin, Closeable
/**
* This method creates a directory at the specified path.
+ * If the directory already exist, this will be a no-op.
*
* @param path
* The path where the directory needs to be created.
@@ -166,7 +175,9 @@ public interface BackupRepository extends NamedListInitializedPlugin, Closeable
* @throws IOException
* in case of errors
*/
- void copyFileFrom(Directory sourceDir, String fileName, URI dest) throws IOException;
+ default void copyFileFrom(Directory sourceDir, String fileName, URI dest) throws IOException {
+ copyIndexFileFrom(sourceDir, fileName, dest, fileName);
+ }
/**
* Copy a file from specified <code>sourceRepo</code> to the destination directory (i.e. restore).
@@ -180,5 +191,89 @@ public interface BackupRepository extends NamedListInitializedPlugin, Closeable
* @throws IOException
* in case of errors.
*/
- void copyFileTo(URI sourceRepo, String fileName, Directory dest) throws IOException;
+ default void copyFileTo(URI sourceRepo, String fileName, Directory dest) throws IOException {
+ copyIndexFileTo(sourceRepo, fileName, dest, fileName);
+ }
+
+ /**
+ * List all files or directories directly under {@code path}.
+ * @return an empty array in case of IOException
+ */
+ default String[] listAllOrEmpty(URI path) {
+ try {
+ return this.listAll(path);
+ } catch (IOException e) {
+ return new String[0];
+ }
+ }
+
+ default void copyIndexFileFrom(Directory sourceDir, String sourceFileName, Directory destDir, String destFileName) throws IOException {
+ boolean success = false;
+ try (ChecksumIndexInput is = sourceDir.openChecksumInput(sourceFileName, DirectoryFactory.IOCONTEXT_NO_CACHE);
+ IndexOutput os = destDir.createOutput(destFileName, DirectoryFactory.IOCONTEXT_NO_CACHE)) {
+ os.copyBytes(is, is.length() - CodecUtil.footerLength());
+
+ // ensure that index file is not corrupted
+ CodecUtil.checkFooter(is);
+ CodecUtil.writeFooter(os);
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.deleteFilesIgnoringExceptions(destDir, destFileName);
+ }
+ }
+ }
+
+ /**
+ * Delete {@code files} at {@code path}
+ * @since 8.3.0
+ */
+ default void delete(URI path, Collection<String> files, boolean ignoreNoSuchFileException) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Get checksum of {@code fileName} at {@code dir}.
+ * This method only be called on Lucene index files
+ * @since 8.3.0
+ */
+ default Checksum checksum(Directory dir, String fileName) throws IOException {
+ try (IndexInput in = dir.openChecksumInput(fileName, IOContext.READONCE)) {
+ return new Checksum(CodecUtil.retrieveChecksum(in), in.length());
+ }
+ }
+
+ /**
+ * Copy an index file from specified <code>sourceDir</code> to the destination repository (i.e. backup).
+ *
+ * @param sourceDir
+ * The source directory hosting the file to be copied.
+ * @param sourceFileName
+ * The name of the file to by copied
+ * @param destDir
+ * The destination backup location.
+ * @throws IOException
+ * in case of errors
+ * @throws CorruptIndexException
+ * in case checksum of the file does not match with precomputed checksum stored at the end of the file
+ * @since 8.3.0
+ */
+ default void copyIndexFileFrom(Directory sourceDir, String sourceFileName, URI destDir, String destFileName) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Copy an index file from specified <code>sourceRepo</code> to the destination directory (i.e. restore).
+ *
+ * @param sourceRepo
+ * The source URI hosting the file to be copied.
+ * @param dest
+ * The destination where the file should be copied.
+ * @throws IOException
+ * in case of errors.
+ * @since 8.3.0
+ */
+ default void copyIndexFileTo(URI sourceRepo, String sourceFileName, Directory dest, String destFileName) throws IOException {
+ throw new UnsupportedOperationException();
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java b/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java
index 9e02b21c109..6d57a90e856 100644
--- a/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java
@@ -60,18 +60,24 @@ public class BackupRepositoryFactory {
if (this.defaultBackupRepoPlugin != null) {
log.info("Default configuration for backup repository is with configuration params {}",
- defaultBackupRepoPlugin);
+ defaultBackupRepoPlugin);
}
}
}
+ @SuppressWarnings({"unchecked"})
public BackupRepository newInstance(SolrResourceLoader loader, String name) {
Objects.requireNonNull(loader);
Objects.requireNonNull(name);
PluginInfo repo = Objects.requireNonNull(backupRepoPluginByName.get(name),
- "Could not find a backup repository with name " + name);
+ "Could not find a backup repository with name " + name);
BackupRepository result = loader.newInstance(repo.className, BackupRepository.class);
+ if ("trackingBackupRepository".equals(name) && repo.initArgs.get("factory") == null) {
+ repo.initArgs.add("factory", this);
+ repo.initArgs.add("loader", loader);
+ }
+
result.init(repo.initArgs);
return result;
}
diff --git a/solr/core/src/java/org/apache/solr/core/backup/repository/HdfsBackupRepository.java b/solr/core/src/java/org/apache/solr/core/backup/repository/HdfsBackupRepository.java
index ada0b57a29f..b1f1c5e4bbb 100644
--- a/solr/core/src/java/org/apache/solr/core/backup/repository/HdfsBackupRepository.java
+++ b/solr/core/src/java/org/apache/solr/core/backup/repository/HdfsBackupRepository.java
@@ -21,6 +21,8 @@ import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;
+import java.nio.file.NoSuchFileException;
+import java.util.Collection;
import java.util.Objects;
import java.lang.invoke.MethodHandles;
@@ -194,18 +196,33 @@ public class HdfsBackupRepository implements BackupRepository {
}
@Override
- public void copyFileFrom(Directory sourceDir, String fileName, URI dest) throws IOException {
- try (HdfsDirectory dir = new HdfsDirectory(new Path(dest), NoLockFactory.INSTANCE,
- hdfsConfig, copyBufferSize)) {
- dir.copyFrom(sourceDir, fileName, fileName, DirectoryFactory.IOCONTEXT_NO_CACHE);
+ public void copyIndexFileFrom(Directory sourceDir, String sourceFileName, URI destDir, String destFileName) throws IOException {
+ try (HdfsDirectory dir = new HdfsDirectory(new Path(destDir), NoLockFactory.INSTANCE,
+ hdfsConfig, copyBufferSize)) {
+ copyIndexFileFrom(sourceDir, sourceFileName, dir, destFileName);
}
}
@Override
- public void copyFileTo(URI sourceRepo, String fileName, Directory dest) throws IOException {
+ public void copyIndexFileTo(URI sourceRepo, String sourceFileName, Directory dest, String destFileName) throws IOException {
try (HdfsDirectory dir = new HdfsDirectory(new Path(sourceRepo), NoLockFactory.INSTANCE,
- hdfsConfig, copyBufferSize)) {
- dest.copyFrom(dir, fileName, fileName, DirectoryFactory.IOCONTEXT_NO_CACHE);
+ hdfsConfig, copyBufferSize)) {
+ dest.copyFrom(dir, sourceFileName, destFileName, DirectoryFactory.IOCONTEXT_NO_CACHE);
}
}
+
+ @Override
+ public void delete(URI path, Collection<String> files, boolean ignoreNoSuchFileException) throws IOException {
+ if (files.isEmpty())
+ return;
+
+ for (String file : files) {
+ Path filePath = new Path(new Path(path), file);
+ boolean success = fileSystem.delete(filePath, false);
+ if (!ignoreNoSuchFileException && !success) {
+ throw new NoSuchFileException(filePath.toString());
+ }
+ }
+ }
+
}
diff --git a/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java b/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java
index 612a61fd4bf..f16db6d693f 100644
--- a/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java
+++ b/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java
@@ -23,10 +23,13 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Collection;
import java.util.Objects;
import org.apache.lucene.store.Directory;
@@ -83,7 +86,13 @@ public class LocalFileSystemRepository implements BackupRepository {
Path result = Paths.get(baseUri);
for (int i = 0; i < pathComponents.length; i++) {
- result = result.resolve(pathComponents[i]);
+ try {
+ result = result.resolve(pathComponents[i]);
+ } catch (Exception e) {
+ // unlikely to happen
+ throw new RuntimeException(e);
+ }
+
}
return result.toUri();
@@ -91,7 +100,10 @@ public class LocalFileSystemRepository implements BackupRepository {
@Override
public void createDirectory(URI path) throws IOException {
- Files.createDirectory(Paths.get(path));
+ Path p = Paths.get(path);
+ if (!Files.exists(p, LinkOption.NOFOLLOW_LINKS)) {
+ Files.createDirectory(p);
+ }
}
@Override
@@ -130,6 +142,12 @@ public class LocalFileSystemRepository implements BackupRepository {
@Override
public String[] listAll(URI dirPath) throws IOException {
+ // It is better to check the existence of the directory first since
+ // creating a FSDirectory will create a corresponds folder if the directory does not exist
+ if (!exists(dirPath)) {
+ return new String[0];
+ }
+
try (FSDirectory dir = new NIOFSDirectory(Paths.get(dirPath), NoLockFactory.INSTANCE)) {
return dir.listAll();
}
@@ -141,16 +159,33 @@ public class LocalFileSystemRepository implements BackupRepository {
}
@Override
- public void copyFileFrom(Directory sourceDir, String fileName, URI dest) throws IOException {
- try (FSDirectory dir = new NIOFSDirectory(Paths.get(dest), NoLockFactory.INSTANCE)) {
- dir.copyFrom(sourceDir, fileName, fileName, DirectoryFactory.IOCONTEXT_NO_CACHE);
+ public void copyIndexFileFrom(Directory sourceDir, String sourceFileName, URI destDir, String destFileName) throws IOException {
+ try (FSDirectory dir = new NIOFSDirectory(Paths.get(destDir), NoLockFactory.INSTANCE)) {
+ copyIndexFileFrom(sourceDir, sourceFileName, dir, destFileName);
}
}
@Override
- public void copyFileTo(URI sourceDir, String fileName, Directory dest) throws IOException {
+ public void copyIndexFileTo(URI sourceDir, String sourceFileName, Directory dest, String destFileName) throws IOException {
try (FSDirectory dir = new NIOFSDirectory(Paths.get(sourceDir), NoLockFactory.INSTANCE)) {
- dest.copyFrom(dir, fileName, fileName, DirectoryFactory.IOCONTEXT_NO_CACHE);
+ dest.copyFrom(dir, sourceFileName, destFileName, DirectoryFactory.IOCONTEXT_NO_CACHE);
+ }
+ }
+
+ @Override
+ public void delete(URI path, Collection<String> files, boolean ignoreNoSuchFileException) throws IOException {
+ if (files.isEmpty())
+ return;
+
+ try (FSDirectory dir = new NIOFSDirectory(Paths.get(path), NoLockFactory.INSTANCE)) {
+ for (String file : files) {
+ try {
+ dir.deleteFile(file);
+ } catch (NoSuchFileException e) {
+ if (!ignoreNoSuchFileException)
+ throw e;
+ }
+ }
}
}
diff --git a/solr/core/src/java/org/apache/solr/handler/IncrementalShardBackup.java b/solr/core/src/java/org/apache/solr/handler/IncrementalShardBackup.java
new file mode 100644
index 00000000000..49ce0537bfb
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/IncrementalShardBackup.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler;
+
+import org.apache.commons.math3.util.Precision;
+import org.apache.lucene.index.IndexCommit;
+import org.apache.lucene.store.Directory;
+import org.apache.solr.cloud.CloudDescriptor;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.DirectoryFactory;
+import org.apache.solr.core.IndexDeletionPolicyWrapper;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.backup.BackupFilePaths;
+import org.apache.solr.core.backup.Checksum;
+import org.apache.solr.core.backup.ShardBackupId;
+import org.apache.solr.core.backup.ShardBackupMetadata;
+import org.apache.solr.core.backup.repository.BackupRepository;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.URI;
+import java.time.Instant;
+import java.util.Collection;
+import java.util.Optional;
+import java.util.UUID;
+
+/**
+ * Responsible for orchestrating the actual incremental backup process.
+ *
+ * If this is the first backup for a collection, all files are uploaded. But if previous backups exist, uses the most recent
+ * {@link ShardBackupMetadata} file to determine which files already exist in the repository and can be skipped.
+ */
+public class IncrementalShardBackup {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private SolrCore solrCore;
+
+ private BackupFilePaths incBackupFiles;
+ private BackupRepository backupRepo;
+
+ private ShardBackupId prevShardBackupId;
+ private ShardBackupId shardBackupId;
+ private Optional<String> commitNameOption;
+
+ /**
+ *
+ * @param prevShardBackupId previous ShardBackupMetadata file which will be used for skipping
+ * uploading index files already present in this file.
+ * @param shardBackupId file where all meta data of this backup will be stored to.
+ */
+ public IncrementalShardBackup(BackupRepository backupRepo, SolrCore solrCore, BackupFilePaths incBackupFiles,
+ ShardBackupId prevShardBackupId, ShardBackupId shardBackupId,
+ Optional<String> commitNameOption) {
+ this.backupRepo = backupRepo;
+ this.solrCore = solrCore;
+ this.incBackupFiles = incBackupFiles;
+ this.prevShardBackupId = prevShardBackupId;
+ this.shardBackupId = shardBackupId;
+ this.commitNameOption = commitNameOption;
+ }
+
+ @SuppressWarnings({"rawtypes"})
+ public NamedList backup() throws Exception {
+ final IndexCommit indexCommit = getAndSaveIndexCommit();
+ try {
+ return backup(indexCommit);
+ } finally {
+ solrCore.getDeletionPolicy().releaseCommitPoint(indexCommit.getGeneration());
+ }
+ }
+
+ /**
+ * Returns {@link IndexDeletionPolicyWrapper#getAndSaveLatestCommit} unless a particular commitName was requested.
+ * <p>
+ * Note:
+ * <ul>
+ * <li>This method does error handling when the commit can't be found and wraps them in {@link SolrException}
+ * </li>
+ * <li>If this method returns, the result will be non null, and the caller <em>MUST</em>
+ * call {@link IndexDeletionPolicyWrapper#releaseCommitPoint} when finished
+ * </li>
+ * </ul>
+ */
+ private IndexCommit getAndSaveIndexCommit() throws IOException {
+ if (commitNameOption.isPresent()) {
+ return SnapShooter.getAndSaveNamedIndexCommit(solrCore, commitNameOption.get());
+ }
+
+ final IndexDeletionPolicyWrapper delPolicy = solrCore.getDeletionPolicy();
+ final IndexCommit commit = delPolicy.getAndSaveLatestCommit();
+ if (null == commit) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Index does not yet have any commits for core " +
+ solrCore.getName());
+ }
+ if (log.isDebugEnabled()) {
+ log.debug("Using latest commit: generation={}", commit.getGeneration());
+ }
+ return commit;
+ }
+
+ private IndexCommit getAndSaveLatestIndexCommit(IndexDeletionPolicyWrapper delPolicy) {
+ final IndexCommit commit = delPolicy.getAndSaveLatestCommit();
+ if (null == commit) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Index does not yet have any commits for core " +
+ solrCore.getName());
+ }
+ if (log.isDebugEnabled()) {
+ log.debug("Using latest commit: generation={}", commit.getGeneration());
+ }
+ return commit;
+ }
+
+ // note: remember to reserve the indexCommit first so it won't get deleted concurrently
+ @SuppressWarnings({"rawtypes"})
+ protected NamedList backup(final IndexCommit indexCommit) throws Exception {
+ assert indexCommit != null;
+ URI backupLocation = incBackupFiles.getBackupLocation();
+ log.info("Creating backup snapshot at {} shardBackupMetadataFile:{}", backupLocation, shardBackupId);
+ NamedList<Object> details = new NamedList<>();
+ details.add("startTime", Instant.now().toString());
+
+ Collection<String> files = indexCommit.getFileNames();
+ Directory dir = solrCore.getDirectoryFactory().get(solrCore.getIndexDir(),
+ DirectoryFactory.DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType);
+ try {
+ BackupStats stats = incrementalCopy(files, dir);
+ details.add("indexFileCount", stats.fileCount);
+ details.add("uploadedIndexFileCount", stats.uploadedFileCount);
+ details.add("indexSizeMB", stats.getIndexSizeMB());
+ details.add("uploadedIndexFileMB", stats.getTotalUploadedMB());
+ } finally {
+ solrCore.getDirectoryFactory().release(dir);
+ }
+
+ CloudDescriptor cd = solrCore.getCoreDescriptor().getCloudDescriptor();
+ if (cd != null) {
+ details.add("shard", cd.getShardId());
+ }
+
+ details.add("endTime", Instant.now().toString());
+ details.add("shardBackupId", shardBackupId.getIdAsString());
+ log.info("Done creating backup snapshot at {} shardBackupMetadataFile:{}", backupLocation, shardBackupId);
+ return details;
+ }
+
+ private ShardBackupMetadata getPrevBackupPoint() throws IOException {
+ if (prevShardBackupId == null) {
+ return ShardBackupMetadata.empty();
+ }
+ return ShardBackupMetadata.from(backupRepo, incBackupFiles.getShardBackupMetadataDir(), prevShardBackupId);
+ }
+
+ private BackupStats incrementalCopy(Collection<String> indexFiles, Directory dir) throws IOException {
+ ShardBackupMetadata oldBackupPoint = getPrevBackupPoint();
+ ShardBackupMetadata currentBackupPoint = ShardBackupMetadata.empty();
+ URI indexDir = incBackupFiles.getIndexDir();
+ BackupStats backupStats = new BackupStats();
+
+ for(String fileName : indexFiles) {
+ Optional<ShardBackupMetadata.BackedFile> opBackedFile = oldBackupPoint.getFile(fileName);
+ Checksum originalFileCS = backupRepo.checksum(dir, fileName);
+
+ if (opBackedFile.isPresent()) {
+ ShardBackupMetadata.BackedFile backedFile = opBackedFile.get();
+ Checksum existedFileCS = backedFile.fileChecksum;
+ if (existedFileCS.equals(originalFileCS)) {
+ currentBackupPoint.addBackedFile(opBackedFile.get());
+ backupStats.skippedUploadingFile(existedFileCS);
+ continue;
+ }
+ }
+
+ String backedFileName = UUID.randomUUID().toString();
+ backupRepo.copyIndexFileFrom(dir, fileName, indexDir, backedFileName);
+
+ currentBackupPoint.addBackedFile(backedFileName, fileName, originalFileCS);
+ backupStats.uploadedFile(originalFileCS);
+ }
+
+ currentBackupPoint.store(backupRepo, incBackupFiles.getShardBackupMetadataDir(), shardBackupId);
+ return backupStats;
+ }
+
+ private static class BackupStats {
+ private int fileCount;
+ private int uploadedFileCount;
+ private long indexSize;
+ private long totalUploadedBytes;
+
+ public void uploadedFile(Checksum file) {
+ fileCount++;
+ uploadedFileCount++;
+ indexSize += file.size;
+ totalUploadedBytes += file.size;
+ }
+
+ public void skippedUploadingFile(Checksum existedFile) {
+ fileCount++;
+ indexSize += existedFile.size;
+ }
+
+ public double getIndexSizeMB() {
+ return Precision.round(indexSize / (1024.0 * 1024), 3);
+ }
+
+ public double getTotalUploadedMB() {
+ return Precision.round(totalUploadedBytes / (1024.0 * 1024), 3);
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
index 9908292d91b..71ebd4952c4 100644
--- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
@@ -523,7 +523,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
name = "snapshot." + name;
}
- RestoreCore restoreCore = new RestoreCore(repo, core, locationUri, name);
+ RestoreCore restoreCore = RestoreCore.create(repo, core, locationUri, name);
try {
MDC.put("RestoreCore.core", core.getName());
MDC.put("RestoreCore.backupLocation", location);
diff --git a/solr/core/src/java/org/apache/solr/handler/RestoreCore.java b/solr/core/src/java/org/apache/solr/handler/RestoreCore.java
index 3e12d4b101c..25b559eb5da 100644
--- a/solr/core/src/java/org/apache/solr/handler/RestoreCore.java
+++ b/solr/core/src/java/org/apache/solr/handler/RestoreCore.java
@@ -16,11 +16,16 @@
*/
package org.apache.solr.handler;
+import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.URI;
import java.text.SimpleDateFormat;
+import java.util.Arrays;
import java.util.Date;
+import java.util.HashSet;
import java.util.Locale;
+import java.util.Optional;
+import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
@@ -31,6 +36,10 @@ import org.apache.lucene.store.IndexInput;
import org.apache.solr.common.SolrException;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.backup.BackupFilePaths;
+import org.apache.solr.core.backup.Checksum;
+import org.apache.solr.core.backup.ShardBackupId;
+import org.apache.solr.core.backup.ShardBackupMetadata;
import org.apache.solr.core.backup.repository.BackupRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,16 +48,25 @@ public class RestoreCore implements Callable<Boolean> {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private final String backupName;
- private final URI backupLocation;
private final SolrCore core;
- private final BackupRepository backupRepo;
+ private RestoreRepository repository;
- public RestoreCore(BackupRepository backupRepo, SolrCore core, URI location, String name) {
- this.backupRepo = backupRepo;
+ private RestoreCore(SolrCore core, RestoreRepository repository) {
this.core = core;
- this.backupLocation = location;
- this.backupName = name;
+ this.repository = repository;
+ }
+
+ public static RestoreCore create(BackupRepository backupRepo, SolrCore core, URI location, String backupname) {
+ RestoreRepository repository = new BasicRestoreRepository(backupRepo.resolve(location, backupname), backupRepo);
+ return new RestoreCore(core, repository);
+ }
+
+ public static RestoreCore createWithMetaFile(BackupRepository repo, SolrCore core, URI location, ShardBackupId shardBackupId) throws IOException {
+ BackupFilePaths incBackupFiles = new BackupFilePaths(repo, location);
+ URI shardBackupMetadataDir = incBackupFiles.getShardBackupMetadataDir();
+ ShardBackupIdRestoreRepository resolver = new ShardBackupIdRestoreRepository(location, incBackupFiles.getIndexDir(),
+ repo, ShardBackupMetadata.from(repo, shardBackupMetadataDir, shardBackupId));
+ return new RestoreCore(core, resolver);
}
@Override
@@ -57,8 +75,6 @@ public class RestoreCore implements Callable<Boolean> {
}
public boolean doRestore() throws Exception {
-
- URI backupPath = backupRepo.resolve(backupLocation, backupName);
SimpleDateFormat dateFormat = new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT);
String restoreIndexName = "restore." + dateFormat.format(new Date());
String restoreIndexPath = core.getDataDir() + restoreIndexName;
@@ -69,31 +85,34 @@ public class RestoreCore implements Callable<Boolean> {
try {
restoreIndexDir = core.getDirectoryFactory().get(restoreIndexPath,
- DirectoryFactory.DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType);
+ DirectoryFactory.DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType);
//Prefer local copy.
indexDir = core.getDirectoryFactory().get(indexDirPath,
- DirectoryFactory.DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType);
-
+ DirectoryFactory.DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType);
+ Set<String> indexDirFiles = new HashSet<>(Arrays.asList(indexDir.listAll()));
//Move all files from backupDir to restoreIndexDir
- for (String filename : backupRepo.listAll(backupPath)) {
+ for (String filename : repository.listAllFiles()) {
checkInterrupted();
- log.info("Copying file {} to restore directory ", filename);
- try (IndexInput indexInput = backupRepo.openInput(backupPath, filename, IOContext.READONCE)) {
- Long checksum = null;
- try {
- checksum = CodecUtil.retrieveChecksum(indexInput);
- } catch (Exception e) {
- log.warn("Could not read checksum from index file: {}", filename, e);
- }
- long length = indexInput.length();
- IndexFetcher.CompareResult compareResult = IndexFetcher.compareFile(indexDir, filename, length, checksum);
- if (!compareResult.equal ||
- (IndexFetcher.filesToAlwaysDownloadIfNoChecksums(filename, length, compareResult))) {
- backupRepo.copyFileTo(backupPath, filename, restoreIndexDir);
+ try {
+ if (indexDirFiles.contains(filename)) {
+ Checksum cs = repository.checksum(filename);
+ IndexFetcher.CompareResult compareResult;
+ if (cs == null) {
+ compareResult = new IndexFetcher.CompareResult();
+ compareResult.equal = false;
+ } else {
+ compareResult = IndexFetcher.compareFile(indexDir, filename, cs.size, cs.checksum);
+ }
+ if (!compareResult.equal ||
+ (IndexFetcher.filesToAlwaysDownloadIfNoChecksums(filename, cs.size, compareResult))) {
+ repository.repoCopy(filename, restoreIndexDir);
+ } else {
+ //prefer local copy
+ repository.localCopy(indexDir, filename, restoreIndexDir);
+ }
} else {
- //prefer local copy
- restoreIndexDir.copyFrom(indexDir, filename, filename, IOContext.READONCE);
+ repository.repoCopy(filename, restoreIndexDir);
}
} catch (Exception e) {
log.warn("Exception while restoring the backup index ", e);
@@ -115,7 +134,7 @@ public class RestoreCore implements Callable<Boolean> {
Directory dir = null;
try {
dir = core.getDirectoryFactory().get(core.getDataDir(), DirectoryFactory.DirContext.META_DATA,
- core.getSolrConfig().indexConfig.lockType);
+ core.getSolrConfig().indexConfig.lockType);
dir.deleteFile(IndexFetcher.INDEX_PROPERTIES);
} finally {
if (dir != null) {
@@ -160,4 +179,106 @@ public class RestoreCore implements Callable<Boolean> {
waitSearcher[0].get();
}
}
+
+ /**
+ * A minimal version of {@link BackupRepository} used for restoring
+ */
+ private interface RestoreRepository {
+ String[] listAllFiles() throws IOException;
+ IndexInput openInput(String filename) throws IOException;
+ void repoCopy(String filename, Directory dest) throws IOException;
+ void localCopy(Directory src, String filename, Directory dest) throws IOException;
+ Checksum checksum(String filename) throws IOException;
+ }
+
+ /**
+ * A basic {@link RestoreRepository} simply delegates all calls to {@link BackupRepository}
+ */
+ private static class BasicRestoreRepository implements RestoreRepository {
+ protected final URI backupPath;
+ protected final BackupRepository repository;
+
+ public BasicRestoreRepository(URI backupPath, BackupRepository repository) {
+ this.backupPath = backupPath;
+ this.repository = repository;
+ }
+
+ public String[] listAllFiles() throws IOException {
+ return repository.listAll(backupPath);
+ }
+
+ public IndexInput openInput(String filename) throws IOException {
+ return repository.openInput(backupPath, filename, IOContext.READONCE);
+ }
+
+ public void repoCopy(String filename, Directory dest) throws IOException {
+ repository.copyFileTo(backupPath, filename, dest);
+ }
+
+ public void localCopy(Directory src, String filename, Directory dest) throws IOException {
+ dest.copyFrom(src, filename, filename, IOContext.READONCE);
+ }
+
+ public Checksum checksum(String filename) throws IOException {
+ try (IndexInput indexInput = repository.openInput(backupPath, filename, IOContext.READONCE)) {
+ try {
+ long checksum = CodecUtil.retrieveChecksum(indexInput);
+ long length = indexInput.length();
+ return new Checksum(checksum, length);
+ } catch (Exception e) {
+ log.warn("Could not read checksum from index file: {}", filename, e);
+ }
+ }
+ return null;
+ }
+ }
+
+ /**
+ * A {@link RestoreRepository} based on information stored in {@link ShardBackupMetadata}
+ */
+ private static class ShardBackupIdRestoreRepository implements RestoreRepository {
+
+ private final ShardBackupMetadata shardBackupMetadata;
+ private final URI indexURI;
+ protected final URI backupPath;
+ protected final BackupRepository repository;
+
+ public ShardBackupIdRestoreRepository(URI backupPath, URI indexURI, BackupRepository repository, ShardBackupMetadata shardBackupMetadata) {
+ this.shardBackupMetadata = shardBackupMetadata;
+ this.indexURI = indexURI;
+ this.backupPath = backupPath;
+ this.repository = repository;
+ }
+
+ @Override
+ public String[] listAllFiles() {
+ return shardBackupMetadata.listOriginalFileNames().toArray(new String[0]);
+ }
+
+ @Override
+ public IndexInput openInput(String filename) throws IOException {
+ String storedFileName = getStoredFilename(filename);
+ return repository.openInput(indexURI, storedFileName, IOContext.READONCE);
+ }
+
+ @Override
+ public void repoCopy(String filename, Directory dest) throws IOException {
+ String storedFileName = getStoredFilename(filename);
+ repository.copyIndexFileTo(this.indexURI, storedFileName, dest, filename);
+ }
+
+ @Override
+ public void localCopy(Directory src, String filename, Directory dest) throws IOException {
+ dest.copyFrom(src, filename, filename, IOContext.READONCE);
+ }
+
+ public Checksum checksum(String filename) {
+ Optional<ShardBackupMetadata.BackedFile> backedFile = shardBackupMetadata.getFile(filename);
+ return backedFile.map(bf -> bf.fileChecksum).orElse(null);
+ }
+
+ private String getStoredFilename(String filename) {
+ return shardBackupMetadata.getFile(filename).get().uniqueFileName;
+ }
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/handler/SnapShooter.java b/solr/core/src/java/org/apache/solr/handler/SnapShooter.java
index 8637316cfac..4b4a1491d87 100644
--- a/solr/core/src/java/org/apache/solr/handler/SnapShooter.java
+++ b/solr/core/src/java/org/apache/solr/handler/SnapShooter.java
@@ -180,25 +180,7 @@ public class SnapShooter {
private IndexCommit getAndSaveIndexCommit() throws IOException {
final IndexDeletionPolicyWrapper delPolicy = solrCore.getDeletionPolicy();
if (null != commitName) {
- final SolrSnapshotMetaDataManager snapshotMgr = solrCore.getSnapshotMetaDataManager();
- // We're going to tell the delPolicy to "save" this commit -- even though it's a named snapshot
- // that will already be protected -- just in case another thread deletes the name.
- // Because of this, we want to sync on the delPolicy to ensure there is no window of time after
- // snapshotMgr confirms commitName exists, but before we have a chance to 'save' it, when
- // the commitName might be deleted *and* the IndexWriter might call onCommit()
- synchronized (delPolicy) {
- final Optional<IndexCommit> namedCommit = snapshotMgr.getIndexCommitByName(commitName);
- if (namedCommit.isPresent()) {
- final IndexCommit commit = namedCommit.get();
- if (log.isDebugEnabled()) {
- log.debug("Using named commit: name={}, generation={}", commitName, commit.getGeneration());
- }
- delPolicy.saveCommitPoint(commit.getGeneration());
- return commit;
- }
- } // else...
- throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to find an index commit with name " +
- commitName + " for core " + solrCore.getName());
+ return getAndSaveNamedIndexCommit(solrCore, commitName);
}
// else: not a named commit...
final IndexCommit commit = delPolicy.getAndSaveLatestCommit();
@@ -212,6 +194,29 @@ public class SnapShooter {
return commit;
}
+ public static IndexCommit getAndSaveNamedIndexCommit(SolrCore solrCore, String commitName) throws IOException {
+ final IndexDeletionPolicyWrapper delPolicy = solrCore.getDeletionPolicy();
+ final SolrSnapshotMetaDataManager snapshotMgr = solrCore.getSnapshotMetaDataManager();
+ // We're going to tell the delPolicy to "save" this commit -- even though it's a named snapshot
+ // that will already be protected -- just in case another thread deletes the name.
+ // Because of this, we want to sync on the delPolicy to ensure there is no window of time after
+ // snapshotMgr confirms commitName exists, but before we have a chance to 'save' it, when
+ // the commitName might be deleted *and* the IndexWriter might call onCommit()
+ synchronized (delPolicy) {
+ final Optional<IndexCommit> namedCommit = snapshotMgr.getIndexCommitByName(commitName);
+ if (namedCommit.isPresent()) {
+ final IndexCommit commit = namedCommit.get();
+ if (log.isDebugEnabled()) {
+ log.debug("Using named commit: name={}, generation={}", commitName, commit.getGeneration());
+ }
+ delPolicy.saveCommitPoint(commit.getGeneration());
+ return commit;
+ }
+ } // else...
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to find an index commit with name " +
+ commitName + " for core " + solrCore.getName());
+ }
+
@SuppressWarnings({"unchecked", "rawtypes"})
public void createSnapAsync(final int numberToKeep, Consumer<NamedList> result) throws IOException {
//TODO should use Solr's ExecutorUtil
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java b/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java
index c295ad5d22d..c2c78066a74 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java
@@ -17,57 +17,100 @@
package org.apache.solr.handler.admin;
-import java.net.URI;
-
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.backup.BackupFilePaths;
+import org.apache.solr.core.backup.ShardBackupId;
import org.apache.solr.core.backup.repository.BackupRepository;
+import org.apache.solr.handler.IncrementalShardBackup;
import org.apache.solr.handler.SnapShooter;
-import static org.apache.solr.common.params.CommonParams.NAME;
-
+import java.net.URI;
+import java.nio.file.Paths;
+import java.util.Optional;
class BackupCoreOp implements CoreAdminHandler.CoreAdminOp {
+
@Override
public void execute(CoreAdminHandler.CallInfo it) throws Exception {
final SolrParams params = it.req.getParams();
String cname = params.required().get(CoreAdminParams.CORE);
- String name = params.required().get(NAME);
-
+ boolean incremental = isIncrementalBackup(params);
+ final String name = parseBackupName(params);
+ final ShardBackupId shardBackupId = parseShardBackupId(params);
+ String prevShardBackupIdStr = params.get(CoreAdminParams.PREV_SHARD_BACKUP_ID, null);
String repoName = params.get(CoreAdminParams.BACKUP_REPOSITORY);
- BackupRepository repository = it.handler.coreContainer.newBackupRepository(repoName);
-
- String location = repository.getBackupLocation(params.get(CoreAdminParams.BACKUP_LOCATION));
- if (location == null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'location' is not specified as a query"
- + " parameter or as a default repository property");
- }
-
// An optional parameter to describe the snapshot to be backed-up. If this
// parameter is not supplied, the latest index commit is backed-up.
String commitName = params.get(CoreAdminParams.COMMIT_NAME);
- URI locationUri = repository.createURI(location);
- try (SolrCore core = it.handler.coreContainer.getCore(cname)) {
- SnapShooter snapShooter = new SnapShooter(repository, core, locationUri, name, commitName);
- // validateCreateSnapshot will create parent dirs instead of throw; that choice is dubious.
- // But we want to throw. One reason is that
- // this dir really should, in fact must, already exist here if triggered via a collection backup on a shared
- // file system. Otherwise, perhaps the FS location isn't shared -- we want an error.
- if (!snapShooter.getBackupRepository().exists(snapShooter.getLocation())) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "Directory to contain snapshots doesn't exist: " + snapShooter.getLocation() + ". " +
- "Note that Backup/Restore of a SolrCloud collection " +
- "requires a shared file system mounted at the same path on all nodes!");
+ try (BackupRepository repository = it.handler.coreContainer.newBackupRepository(repoName);
+ SolrCore core = it.handler.coreContainer.getCore(cname)) {
+ String location = repository.getBackupLocation(params.get(CoreAdminParams.BACKUP_LOCATION));
+ if (location == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'location' is not specified as a query"
+ + " parameter or as a default repository property");
+ }
+
+ URI locationUri = repository.createURI(location);
+
+ if (incremental) {
+ if ("file".equals(locationUri.getScheme())) {
+ core.getCoreContainer().assertPathAllowed(Paths.get(location));
+ }
+ final ShardBackupId prevShardBackupId = prevShardBackupIdStr != null ? ShardBackupId.from(prevShardBackupIdStr) : null;
+ BackupFilePaths incBackupFiles = new BackupFilePaths(repository, locationUri);
+ IncrementalShardBackup incSnapShooter = new IncrementalShardBackup(repository, core, incBackupFiles,
+ prevShardBackupId, shardBackupId, Optional.ofNullable(commitName));
+ @SuppressWarnings({"rawtypes"})
+ NamedList rsp = incSnapShooter.backup();
+ it.rsp.addResponse(rsp);
+ } else {
+ SnapShooter snapShooter = new SnapShooter(repository, core, locationUri, name, commitName);
+ // validateCreateSnapshot will create parent dirs instead of throw; that choice is dubious.
+ // But we want to throw. One reason is that
+ // this dir really should, in fact must, already exist here if triggered via a collection backup on a shared
+ // file system. Otherwise, perhaps the FS location isn't shared -- we want an error.
+ if (!snapShooter.getBackupRepository().exists(snapShooter.getLocation())) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Directory to contain snapshots doesn't exist: " + snapShooter.getLocation() + ". " +
+ "Note that Backup/Restore of a SolrCloud collection " +
+ "requires a shared file system mounted at the same path on all nodes!");
+ }
+ snapShooter.validateCreateSnapshot();
+ snapShooter.createSnapshot();
}
- snapShooter.validateCreateSnapshot();
- snapShooter.createSnapshot();
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "Failed to backup core=" + cname + " because " + e, e);
+ "Failed to backup core=" + cname + " because " + e, e);
+ }
+ }
+
+ /*
+ * 'shardBackupId' is required iff 'incremental=true', but unused otherwise
+ */
+ private ShardBackupId parseShardBackupId(SolrParams params) {
+ if (isIncrementalBackup(params)) {
+ return ShardBackupId.from(params.required().get(CoreAdminParams.SHARD_BACKUP_ID));
}
+ return null;
+ }
+
+ /*
+ * 'name' parameter is required iff 'incremental=false', but unused otherwise.
+ */
+ private String parseBackupName(SolrParams params) {
+ if (isIncrementalBackup(params)) {
+ return null;
+ }
+ return params.required().get(CoreAdminParams.NAME);
+ }
+
+ private boolean isIncrementalBackup(SolrParams params) {
+ return params.getBool(CoreAdminParams.BACKUP_INCREMENTAL, true);
}
}
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index daf82f63d60..32061c04bc3 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -16,14 +16,6 @@
*/
package org.apache.solr.handler.admin;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.net.URI;
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.stream.Collectors;
-
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.apache.commons.io.IOUtils;
@@ -89,62 +81,42 @@ import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.solr.client.solrj.response.RequestStatusState.COMPLETED;
-import static org.apache.solr.client.solrj.response.RequestStatusState.FAILED;
-import static org.apache.solr.client.solrj.response.RequestStatusState.NOT_FOUND;
-import static org.apache.solr.client.solrj.response.RequestStatusState.RUNNING;
-import static org.apache.solr.client.solrj.response.RequestStatusState.SUBMITTED;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+
+import static org.apache.solr.client.solrj.response.RequestStatusState.*;
import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION;
-import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_PREFIX;
-import static org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.CREATE_NODE_SET;
-import static org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.CREATE_NODE_SET_EMPTY;
-import static org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.CREATE_NODE_SET_SHUFFLE;
-import static org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.NUM_SLICES;
-import static org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.ONLY_ACTIVE_NODES;
-import static org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.ONLY_IF_DOWN;
import static org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.REQUESTID;
-import static org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.SHARDS_PROP;
-import static org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.SHARD_UNIQUE;
+import static org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.*;
import static org.apache.solr.cloud.api.collections.RoutedAlias.CREATE_COLLECTION_PREFIX;
import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST;
import static org.apache.solr.common.cloud.DocCollection.DOC_ROUTER;
import static org.apache.solr.common.cloud.DocCollection.PER_REPLICA_STATE;
-import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.NRT_REPLICAS;
-import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_VALUE_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.PULL_REPLICAS;
-import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
-import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_TYPE;
-import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.TLOG_REPLICAS;
-import static org.apache.solr.common.params.CollectionAdminParams.ALIAS;
+import static org.apache.solr.common.cloud.ZkStateReader.*;
import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION;
-import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF;
-import static org.apache.solr.common.params.CollectionAdminParams.COUNT_PROP;
-import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES;
-import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_NAME;
-import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_VALUE;
-import static org.apache.solr.common.params.CollectionAdminParams.SKIP_NODE_ASSIGNMENT;
+import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_PREFIX;
+import static org.apache.solr.common.params.CollectionAdminParams.*;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;
-import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
-import static org.apache.solr.common.params.CommonAdminParams.IN_PLACE_MOVE;
-import static org.apache.solr.common.params.CommonAdminParams.NUM_SUB_SHARDS;
-import static org.apache.solr.common.params.CommonAdminParams.SPLIT_BY_PREFIX;
-import static org.apache.solr.common.params.CommonAdminParams.SPLIT_FUZZ;
-import static org.apache.solr.common.params.CommonAdminParams.SPLIT_METHOD;
-import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE;
+import static org.apache.solr.common.params.CommonAdminParams.*;
import static org.apache.solr.common.params.CommonParams.NAME;
-import static org.apache.solr.common.params.CommonParams.TIMING;
-import static org.apache.solr.common.params.CommonParams.VALUE_LONG;
-import static org.apache.solr.common.params.CoreAdminParams.DATA_DIR;
-import static org.apache.solr.common.params.CoreAdminParams.DELETE_DATA_DIR;
-import static org.apache.solr.common.params.CoreAdminParams.DELETE_INDEX;
-import static org.apache.solr.common.params.CoreAdminParams.DELETE_INSTANCE_DIR;
-import static org.apache.solr.common.params.CoreAdminParams.DELETE_METRICS_HISTORY;
-import static org.apache.solr.common.params.CoreAdminParams.INSTANCE_DIR;
-import static org.apache.solr.common.params.CoreAdminParams.ULOG_DIR;
+import static org.apache.solr.common.params.CommonParams.*;
+import static org.apache.solr.common.params.CoreAdminParams.*;
import static org.apache.solr.common.params.ShardParams._ROUTE_;
import static org.apache.solr.common.util.StrUtils.formatString;
@@ -1057,6 +1029,8 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
}
}
+ boolean incremental = req.getParams().getBool(CoreAdminParams.BACKUP_INCREMENTAL, true);
+
// Check if the specified location is valid for this repository.
final URI uri = repository.createURI(location);
try {
@@ -1064,7 +1038,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
throw new SolrException(ErrorCode.SERVER_ERROR, "specified location " + uri + " does not exist.");
}
} catch (IOException ex) {
- throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to check the existance of " + uri + ". Is it valid?", ex);
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to check the existence of " + uri + ". Is it valid?", ex);
}
String strategy = req.getParams().get(CollectionAdminParams.INDEX_BACKUP_STRATEGY, CollectionAdminParams.COPY_FILES_STRATEGY);
@@ -1072,13 +1046,15 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown index backup strategy " + strategy);
}
- final Map<String, Object> params = copy(req.getParams(), null, NAME, COLLECTION_PROP, FOLLOW_ALIASES, CoreAdminParams.COMMIT_NAME);
+ Map<String, Object> params = copy(req.getParams(), null, NAME, COLLECTION_PROP,
+ FOLLOW_ALIASES, CoreAdminParams.COMMIT_NAME, CoreAdminParams.MAX_NUM_BACKUP_POINTS);
params.put(CoreAdminParams.BACKUP_LOCATION, location);
if (repo != null) {
params.put(CoreAdminParams.BACKUP_REPOSITORY, repo);
}
params.put(CollectionAdminParams.INDEX_BACKUP_STRATEGY, strategy);
+ params.put(CoreAdminParams.BACKUP_INCREMENTAL, incremental);
return params;
}),
RESTORE_OP(RESTORE, (req, rsp, h) -> {
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/RestoreCoreOp.java b/solr/core/src/java/org/apache/solr/handler/admin/RestoreCoreOp.java
index 65214c2226a..7e81f747895 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/RestoreCoreOp.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/RestoreCoreOp.java
@@ -26,6 +26,7 @@ import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.backup.ShardBackupId;
import org.apache.solr.core.backup.repository.BackupRepository;
import org.apache.solr.handler.RestoreCore;
@@ -37,24 +38,29 @@ class RestoreCoreOp implements CoreAdminHandler.CoreAdminOp {
public void execute(CoreAdminHandler.CallInfo it) throws Exception {
final SolrParams params = it.req.getParams();
String cname = params.required().get(CoreAdminParams.CORE);
- String name = params.required().get(NAME);
+ String name = params.get(NAME);
+ String shardBackupIdStr = params.get(CoreAdminParams.SHARD_BACKUP_ID);
+ String repoName = params.get(CoreAdminParams.BACKUP_REPOSITORY);
+
+ if (shardBackupIdStr == null && name == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Either 'name' or 'shardBackupId' must be specified");
+ }
ZkController zkController = it.handler.coreContainer.getZkController();
if (zkController == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Only valid for SolrCloud");
}
- String repoName = params.get(CoreAdminParams.BACKUP_REPOSITORY);
- BackupRepository repository = it.handler.coreContainer.newBackupRepository(repoName);
+ try (BackupRepository repository = it.handler.coreContainer.newBackupRepository(repoName);
+ SolrCore core = it.handler.coreContainer.getCore(cname)) {
- String location = repository.getBackupLocation(params.get(CoreAdminParams.BACKUP_LOCATION));
- if (location == null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'location' is not specified as a query"
- + " parameter or as a default repository property");
- }
+ String location = repository.getBackupLocation(params.get(CoreAdminParams.BACKUP_LOCATION));
+ if (location == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "'location' is not specified as a query"
+ + " parameter or as a default repository property");
+ }
- URI locationUri = repository.createURI(location);
- try (SolrCore core = it.handler.coreContainer.getCore(cname)) {
+ URI locationUri = repository.createURI(location);
CloudDescriptor cd = core.getCoreDescriptor().getCloudDescriptor();
// this core must be the only replica in its shard otherwise
// we cannot guarantee consistency between replicas because when we add data (or restore index) to this replica
@@ -63,7 +69,14 @@ class RestoreCoreOp implements CoreAdminHandler.CoreAdminOp {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Failed to restore core=" + core.getName() + ", the core must be the only replica in its shard");
}
- RestoreCore restoreCore = new RestoreCore(repository, core, locationUri, name);
+
+ RestoreCore restoreCore;
+ if (shardBackupIdStr != null) {
+ final ShardBackupId shardBackupId = ShardBackupId.from(shardBackupIdStr);
+ restoreCore = RestoreCore.createWithMetaFile(repository, core, locationUri, shardBackupId);
+ } else {
+ restoreCore = RestoreCore.create(repository, core, locationUri, name);
+ }
boolean success = restoreCore.doRestore();
if (!success) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Failed to restore core=" + core.getName());
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/HdfsCloudIncrementalBackupTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/HdfsCloudIncrementalBackupTest.java
new file mode 100644
index 00000000000..17fbce04f92
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/HdfsCloudIncrementalBackupTest.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.api.collections;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.solr.cloud.hdfs.HdfsTestUtil;
+import org.apache.solr.common.util.IOUtils;
+import org.apache.solr.util.BadHdfsThreadsFilter;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+@ThreadLeakFilters(defaultFilters = true, filters = {
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+})
+public class HdfsCloudIncrementalBackupTest extends AbstractIncrementalBackupTest{
+ public static final String SOLR_XML = "<solr>\n" +
+ "\n" +
+ " <str name=\"shareSchema\">${shareSchema:false}</str>\n" +
+ " <str name=\"configSetBaseDir\">${configSetBaseDir:configsets}</str>\n" +
+ " <str name=\"coreRootDirectory\">${coreRootDirectory:.}</str>\n" +
+ "\n" +
+ " <shardHandlerFactory name=\"shardHandlerFactory\" class=\"HttpShardHandlerFactory\">\n" +
+ " <str name=\"urlScheme\">${urlScheme:}</str>\n" +
+ " <int name=\"socketTimeout\">${socketTimeout:90000}</int>\n" +
+ " <int name=\"connTimeout\">${connTimeout:15000}</int>\n" +
+ " </shardHandlerFactory>\n" +
+ "\n" +
+ " <solrcloud>\n" +
+ " <str name=\"host\">127.0.0.1</str>\n" +
+ " <int name=\"hostPort\">${hostPort:8983}</int>\n" +
+ " <str name=\"hostContext\">${hostContext:solr}</str>\n" +
+ " <int name=\"zkClientTimeout\">${solr.zkclienttimeout:30000}</int>\n" +
+ " <bool name=\"genericCoreNodeNames\">${genericCoreNodeNames:true}</bool>\n" +
+ " <int name=\"leaderVoteWait\">10000</int>\n" +
+ " <int name=\"distribUpdateConnTimeout\">${distribUpdateConnTimeout:45000}</int>\n" +
+ " <int name=\"distribUpdateSoTimeout\">${distribUpdateSoTimeout:340000}</int>\n" +
+ " </solrcloud>\n" +
+ " \n" +
+ " <backup>\n" +
+ " <repository name=\"trackingBackupRepository\" default=\"true\" class=\"org.apache.solr.core.TrackingBackupRepository\"> \n" +
+ " <str name=\"delegateRepoName\">hdfs</str>\n" +
+ " </repository>\n" +
+ " <repository name=\"hdfs\" class=\"org.apache.solr.core.backup.repository.HdfsBackupRepository\"> \n" +
+ " <str name=\"location\">${solr.hdfs.default.backup.path}</str>\n" +
+ " <str name=\"solr.hdfs.home\">${solr.hdfs.home:}</str>\n" +
+ " <str name=\"solr.hdfs.confdir\">${solr.hdfs.confdir:}</str>\n" +
+ " </repository>\n" +
+ " </backup>\n" +
+ " \n" +
+ "</solr>\n";
+
+ private static MiniDFSCluster dfsCluster;
+ private static String hdfsUri;
+ private static FileSystem fs;
+
+ @BeforeClass
+ public static void setupClass() throws Exception {
+ dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
+ hdfsUri = HdfsTestUtil.getURI(dfsCluster);
+ try {
+ URI uri = new URI(hdfsUri);
+ Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
+ fs = FileSystem.get(uri, conf);
+
+ if (fs instanceof DistributedFileSystem) {
+ // Make sure dfs is not in safe mode
+ while (((DistributedFileSystem) fs).setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_GET, true)) {
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ Thread.interrupted();
+ // continue
+ }
+ }
+ }
+
+ fs.mkdirs(new org.apache.hadoop.fs.Path("/backup"));
+ } catch (IOException | URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+
+ System.setProperty("solr.hdfs.default.backup.path", "/backup");
+ System.setProperty("solr.hdfs.home", hdfsUri + "/solr");
+ useFactory("solr.StandardDirectoryFactory");
+
+ configureCluster(NUM_SHARDS)// nodes
+ .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
+ .withSolrXml(SOLR_XML)
+ .configure();
+ }
+
+ @AfterClass
+ public static void teardownClass() throws Exception {
+ IOUtils.closeQuietly(fs);
+ fs = null;
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ System.clearProperty("solr.hdfs.home");
+ System.clearProperty("solr.hdfs.default.backup.path");
+ System.clearProperty("test.build.data");
+ System.clearProperty("test.cache.data");
+ }
+ }
+
+ @Override
+ public String getCollectionNamePrefix() {
+ return "hdfsbackuprestore";
+ }
+
+ @Override
+ public String getBackupLocation() {
+ return null;
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java
new file mode 100644
index 00000000000..b247ae68925
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.api.collections;
+
+import org.junit.BeforeClass;
+
+public class LocalFSCloudIncrementalBackupTest extends AbstractIncrementalBackupTest {
+ public static final String SOLR_XML = "<solr>\n" +
+ "\n" +
+ " <str name=\"shareSchema\">${shareSchema:false}</str>\n" +
+ " <str name=\"configSetBaseDir\">${configSetBaseDir:configsets}</str>\n" +
+ " <str name=\"coreRootDirectory\">${coreRootDirectory:.}</str>\n" +
+ "\n" +
+ " <shardHandlerFactory name=\"shardHandlerFactory\" class=\"HttpShardHandlerFactory\">\n" +
+ " <str name=\"urlScheme\">${urlScheme:}</str>\n" +
+ " <int name=\"socketTimeout\">${socketTimeout:90000}</int>\n" +
+ " <int name=\"connTimeout\">${connTimeout:15000}</int>\n" +
+ " </shardHandlerFactory>\n" +
+ "\n" +
+ " <solrcloud>\n" +
+ " <str name=\"host\">127.0.0.1</str>\n" +
+ " <int name=\"hostPort\">${hostPort:8983}</int>\n" +
+ " <str name=\"hostContext\">${hostContext:solr}</str>\n" +
+ " <int name=\"zkClientTimeout\">${solr.zkclienttimeout:30000}</int>\n" +
+ " <bool name=\"genericCoreNodeNames\">${genericCoreNodeNames:true}</bool>\n" +
+ " <int name=\"leaderVoteWait\">10000</int>\n" +
+ " <int name=\"distribUpdateConnTimeout\">${distribUpdateConnTimeout:45000}</int>\n" +
+ " <int name=\"distribUpdateSoTimeout\">${distribUpdateSoTimeout:340000}</int>\n" +
+ " </solrcloud>\n" +
+ " \n" +
+ " <backup>\n" +
+ " <repository name=\"trackingBackupRepository\" class=\"org.apache.solr.core.TrackingBackupRepository\"> \n" +
+ " <str name=\"delegateRepoName\">localfs</str>\n" +
+ " </repository>\n" +
+ " <repository name=\"localfs\" class=\"org.apache.solr.core.backup.repository.LocalFileSystemRepository\"> \n" +
+ " </repository>\n" +
+ " </backup>\n" +
+ " \n" +
+ "</solr>\n";
+
+ private static String backupLocation;
+
+ @BeforeClass
+ public static void setupClass() throws Exception {
+ configureCluster(NUM_SHARDS)// nodes
+ .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
+ .withSolrXml(SOLR_XML)
+ .configure();
+
+ boolean whitespacesInPath = random().nextBoolean();
+ if (whitespacesInPath) {
+ backupLocation = createTempDir("my backup").toAbsolutePath().toString();
+ } else {
+ backupLocation = createTempDir("mybackup").toAbsolutePath().toString();
+ }
+ }
+
+ @Override
+ public String getCollectionNamePrefix() {
+ return "backuprestore";
+ }
+
+ @Override
+ public String getBackupLocation() {
+ return backupLocation;
+ }
+
+}
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
index b646e71cbce..424a5c61fd0 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
@@ -24,7 +24,6 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
-import java.util.Properties;
import org.apache.lucene.util.QuickPatchThreadsFilter;
@@ -45,6 +44,7 @@ import org.apache.solr.common.cloud.ZkConfigManager;
import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.backup.BackupManager;
+import org.apache.solr.core.backup.BackupProperties;
import org.apache.solr.core.backup.repository.HdfsBackupRepository;
import org.apache.solr.util.BadHdfsThreadsFilter;
import org.junit.AfterClass;
@@ -53,10 +53,7 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF;
-import static org.apache.solr.core.backup.BackupManager.BACKUP_NAME_PROP;
-import static org.apache.solr.core.backup.BackupManager.BACKUP_PROPS_FILE;
-import static org.apache.solr.core.backup.BackupManager.COLLECTION_NAME_PROP;
+import static org.apache.solr.core.backup.BackupManager.TRADITIONAL_BACKUP_PROPS_FILE;
import static org.apache.solr.core.backup.BackupManager.CONFIG_STATE_DIR;
import static org.apache.solr.core.backup.BackupManager.ZK_STATE_DIR;
@@ -185,6 +182,7 @@ public class TestHdfsCloudBackupRestore extends AbstractCloudBackupRestoreTestCa
CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(collectionName, backupName)
.setRepositoryName(getBackupRepoName())
+ .setIncremental(false)
.setIndexBackupStrategy(CollectionAdminParams.NO_INDEX_BACKUP_STRATEGY);
backup.process(solrClient);
@@ -194,24 +192,24 @@ public class TestHdfsCloudBackupRestore extends AbstractCloudBackupRestoreTestCa
HdfsBackupRepository repo = new HdfsBackupRepository();
repo.init(new NamedList<>(params));
- BackupManager mgr = new BackupManager(repo, solrClient.getZkStateReader());
URI baseLoc = repo.createURI("/backup");
- Properties props = mgr.readBackupProperties(baseLoc, backupName);
+ BackupManager mgr = BackupManager.forRestore(repo, solrClient.getZkStateReader(), repo.resolve(baseLoc, backupName));
+ BackupProperties props = mgr.readBackupProperties();
assertNotNull(props);
- assertEquals(collectionName, props.getProperty(COLLECTION_NAME_PROP));
- assertEquals(backupName, props.getProperty(BACKUP_NAME_PROP));
- assertEquals(configName, props.getProperty(COLL_CONF));
+ assertEquals(collectionName, props.getCollection());
+ assertEquals(backupName, props.getBackupName());
+ assertEquals(configName, props.getConfigName());
- DocCollection collectionState = mgr.readCollectionState(baseLoc, backupName, collectionName);
+ DocCollection collectionState = mgr.readCollectionState(collectionName);
assertNotNull(collectionState);
assertEquals(collectionName, collectionState.getName());
URI configDirLoc = repo.resolve(baseLoc, backupName, ZK_STATE_DIR, CONFIG_STATE_DIR, configName);
assertTrue(repo.exists(configDirLoc));
- Collection<String> expected = Arrays.asList(BACKUP_PROPS_FILE, ZK_STATE_DIR);
+ Collection<String> expected = Arrays.asList(TRADITIONAL_BACKUP_PROPS_FILE, ZK_STATE_DIR);
URI backupLoc = repo.resolve(baseLoc, backupName);
String[] dirs = repo.listAll(backupLoc);
for (String d : dirs) {
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestLocalFSCloudBackupRestore.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestLocalFSCloudBackupRestore.java
index b74080aab00..672ba8cd56a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestLocalFSCloudBackupRestore.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestLocalFSCloudBackupRestore.java
@@ -139,7 +139,7 @@ public class TestLocalFSCloudBackupRestore extends AbstractCloudBackupRestoreTes
public PoinsionedRepository() {
super();
}
- @Override
+ @Override
public void copyFileFrom(Directory sourceDir, String fileName, URI dest) throws IOException {
throw new UnsupportedOperationException(poisioned);
}
diff --git a/solr/core/src/test/org/apache/solr/core/backup/BackupFilePathsTest.java b/solr/core/src/test/org/apache/solr/core/backup/BackupFilePathsTest.java
new file mode 100644
index 00000000000..bd48789195c
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/core/backup/BackupFilePathsTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.core.backup;
+
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Optional;
+
+import static org.junit.Assert.*;
+
+/**
+ * Unit tests for {@link BackupFilePaths}
+ */
+public class BackupFilePathsTest {
+
+ @Test
+ public void testGetBackupPropsName() {
+ final BackupId initialId = BackupId.zero();
+ final BackupId subsequentId = initialId.nextBackupId();
+
+ assertEquals("backup_0.properties", BackupFilePaths.getBackupPropsName(initialId));
+ assertEquals("backup_1.properties", BackupFilePaths.getBackupPropsName(subsequentId));
+ }
+
+ @Test
+ public void testFindAllBackupIdCanReturnEmpty() {
+ final List<BackupId> foundBackupIds = BackupFilePaths.findAllBackupIdsFromFileListing(new String[0]);
+ assertTrue(foundBackupIds.isEmpty());
+ }
+
+ @Test
+ public void testFindAllBackupPropertiesFiles() {
+ final String[] backupFiles = new String[] {"aaa", "baa.properties", "backup.properties", "backup_1.properties",
+ "backup_2.properties", "backup_neqewq.properties", "backup999.properties"};
+ final List<BackupId> foundBackupIds = BackupFilePaths.findAllBackupIdsFromFileListing(backupFiles);
+
+ assertEquals(2, foundBackupIds.size());
+ assertEquals(new BackupId(1), foundBackupIds.get(0));
+ assertEquals(new BackupId(2), foundBackupIds.get(1));
+ }
+
+ @Test
+ public void testFindMostRecentBackupIdCanReturnEmpty() {
+ Optional<BackupId> op = BackupFilePaths.findMostRecentBackupIdFromFileListing(new String[0]);
+ assertFalse(op.isPresent());
+ }
+
+ @Test
+ public void testFindMostRecentBackupPropertiesFile() {
+ final String[] backupFiles = new String[] {"aaa", "baa.properties", "backup.properties", "backup_1.properties",
+ "backup_2.properties", "backup_neqewq.properties", "backup999.properties"};
+ final Optional<BackupId> filenameOption = BackupFilePaths.findMostRecentBackupIdFromFileListing(backupFiles);
+ assertTrue(filenameOption.isPresent());
+ assertEquals(new BackupId(2), filenameOption.get());
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/core/backup/BackupIdTest.java b/solr/core/src/test/org/apache/solr/core/backup/BackupIdTest.java
new file mode 100644
index 00000000000..d4d76fa1e15
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/core/backup/BackupIdTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.core.backup;
+
+import org.apache.solr.SolrTestCase;
+import org.junit.Test;
+
+import static org.apache.solr.core.backup.BackupId.TRADITIONAL_BACKUP;
+
+/**
+ * Unit tests for {@link BackupId}
+ */
+public class BackupIdTest extends SolrTestCase {
+
+ @Test
+ public void testZero() {
+ final BackupId id = BackupId.zero();
+ assertEquals(0, id.getId());
+ }
+
+ @Test
+ public void testTraditionalBackupId() {
+ final BackupId id = BackupId.traditionalBackup();
+ assertEquals(TRADITIONAL_BACKUP, id.getId());
+ }
+
+ @Test
+ public void testBackupIdIncrementing() {
+ final BackupId initialId = new BackupId(3);
+ final BackupId nextId = initialId.nextBackupId();
+
+ assertEquals(3, initialId.getId());
+ assertEquals(4, nextId.getId());
+ assertTrue(initialId.compareTo(nextId) < 0);
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/core/backup/repository/HdfsBackupRepositoryIntegrationTest.java b/solr/core/src/test/org/apache/solr/core/backup/repository/HdfsBackupRepositoryIntegrationTest.java
new file mode 100644
index 00000000000..769ad99c0d6
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/core/backup/repository/HdfsBackupRepositoryIntegrationTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.core.backup.repository;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.solr.cloud.api.collections.AbstractBackupRepositoryTest;
+import org.apache.solr.cloud.hdfs.HdfsTestUtil;
+import org.apache.solr.common.util.IOUtils;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.HdfsDirectoryFactory;
+import org.apache.solr.util.BadHdfsThreadsFilter;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+@ThreadLeakFilters(defaultFilters = true, filters = {
+ BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
+})
+public class HdfsBackupRepositoryIntegrationTest extends AbstractBackupRepositoryTest {
+ private static MiniDFSCluster dfsCluster;
+ private static String hdfsUri;
+ private static FileSystem fs;
+
+ @BeforeClass
+ public static void setupClass() throws Exception {
+ dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
+ hdfsUri = HdfsTestUtil.getURI(dfsCluster);
+ try {
+ URI uri = new URI(hdfsUri);
+ Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster);
+ fs = FileSystem.get(uri, conf);
+
+ if (fs instanceof DistributedFileSystem) {
+ // Make sure dfs is not in safe mode
+ while (((DistributedFileSystem) fs).setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_GET, true)) {
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ Thread.interrupted();
+ // continue
+ }
+ }
+ }
+
+ fs.mkdirs(new org.apache.hadoop.fs.Path("/backup"));
+ } catch (IOException | URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+
+ System.setProperty("solr.hdfs.default.backup.path", "/backup");
+ System.setProperty("solr.hdfs.home", hdfsUri + "/solr");
+ useFactory("solr.StandardDirectoryFactory");
+ }
+
+ @AfterClass
+ public static void teardownClass() throws Exception {
+ IOUtils.closeQuietly(fs);
+ fs = null;
+ try {
+ HdfsTestUtil.teardownClass(dfsCluster);
+ } finally {
+ dfsCluster = null;
+ System.clearProperty("solr.hdfs.home");
+ System.clearProperty("solr.hdfs.default.backup.path");
+ System.clearProperty("test.build.data");
+ System.clearProperty("test.cache.data");
+ }
+ }
+
+ @Override
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ protected BackupRepository getRepository() {
+ HdfsBackupRepository repository = new HdfsBackupRepository();
+ NamedList config = new NamedList();
+ config.add(HdfsDirectoryFactory.HDFS_HOME, hdfsUri + "/solr");
+ repository.init(config);
+ return repository;
+ }
+
+ @Override
+ protected URI getBaseUri() throws URISyntaxException {
+ return new URI(hdfsUri+"/solr/tmp");
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCloudSnapshots.java b/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCloudSnapshots.java
index 162b0f37d04..dad6f4a9328 100644
--- a/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCloudSnapshots.java
+++ b/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCloudSnapshots.java
@@ -50,6 +50,12 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/**
+ * Tests snapshot functionality in a SolrCloud cluster.
+ *
+ * This test uses the (now deprecated) traditional backup method/format. For more thorough tests using the new format,
+ * see {@link org.apache.solr.handler.TestIncrementalCoreBackup}
+ */
@SolrTestCaseJ4.SuppressSSL // Currently unknown why SSL does not work with this test
@Slow
public class TestSolrCloudSnapshots extends SolrCloudTestCase {
@@ -173,7 +179,7 @@ public class TestSolrCloudSnapshots extends SolrCloudTestCase {
//Create a backup using the earlier created snapshot.
{
CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(collectionName, backupName)
- .setLocation(backupLocation).setCommitName(commitName);
+ .setLocation(backupLocation).setCommitName(commitName).setIncremental(false);
if (random().nextBoolean()) {
assertEquals(0, backup.process(solrClient).getStatus());
} else {
diff --git a/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCoreSnapshots.java b/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCoreSnapshots.java
index 18cbb1ceb52..81e72fcab63 100644
--- a/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCoreSnapshots.java
+++ b/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCoreSnapshots.java
@@ -56,6 +56,12 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/**
+ * Tests for index backing up and restoring index snapshots.
+ *
+ * These tests use the deprecated "full-snapshot" based backup method. For tests that cover similar snapshot
+ * functionality incrementally, see {@link org.apache.solr.handler.TestIncrementalCoreBackup}
+ */
@SolrTestCaseJ4.SuppressSSL // Currently unknown why SSL does not work with this test
@Slow
public class TestSolrCoreSnapshots extends SolrCloudTestCase {
@@ -131,6 +137,7 @@ public class TestSolrCoreSnapshots extends SolrCloudTestCase {
params.put("name", backupName);
params.put("commitName", commitName);
params.put("location", location);
+ params.put("incremental", "false");
BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.BACKUPCORE.toString(), params);
}
diff --git a/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java b/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java
index cf9d1c0952a..e4168e3a9bd 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java
@@ -16,13 +16,6 @@
*/
package org.apache.solr.handler;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.HashMap;
-import java.util.Map;
-
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
@@ -48,6 +41,8 @@ import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
+import org.apache.solr.core.backup.BackupId;
+import org.apache.solr.core.backup.ShardBackupId;
import org.apache.solr.util.BadHdfsThreadsFilter;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -55,6 +50,13 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
+
@ThreadLeakFilters(defaultFilters = true, filters = {
SolrIgnoredThreadsFilter.class,
QuickPatchThreadsFilter.class,
@@ -182,6 +184,7 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase {
boolean testViaReplicationHandler = random().nextBoolean();
String baseUrl = cluster.getJettySolrRunners().get(0).getBaseUrl().toString();
+ final String shardBackupId = new ShardBackupId("standalone", BackupId.zero()).getIdAsString();
try (HttpSolrClient leaderClient = getHttpSolrClient(replicaBaseUrl)) {
// Create a backup.
@@ -196,6 +199,7 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase {
Map<String,String> params = new HashMap<>();
params.put("name", backupName);
params.put(CoreAdminParams.BACKUP_REPOSITORY, "hdfs");
+ params.put(CoreAdminParams.SHARD_BACKUP_ID, shardBackupId);
BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.BACKUPCORE.toString(), params);
}
@@ -236,17 +240,23 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase {
Map<String,String> params = new HashMap<>();
params.put("name", "snapshot." + backupName);
params.put(CoreAdminParams.BACKUP_REPOSITORY, "hdfs");
+ params.put(CoreAdminParams.SHARD_BACKUP_ID, shardBackupId);
BackupRestoreUtils.runCoreAdminCommand(replicaBaseUrl, coreName, CoreAdminAction.RESTORECORE.toString(), params);
}
//See if restore was successful by checking if all the docs are present again
BackupRestoreUtils.verifyDocs(nDocs, leaderClient, coreName);
- // Verify the permissions for the backup folder.
- FileStatus status = fs.getFileStatus(new org.apache.hadoop.fs.Path("/backup/snapshot."+backupName));
+ // Verify the permissions on the backup folder.
+ final String backupPath = (testViaReplicationHandler) ?
+ "/backup/snapshot."+ backupName :
+ "/backup/shard_backup_metadata";
+ final FsAction expectedPerms = (testViaReplicationHandler) ? FsAction.ALL : FsAction.READ_EXECUTE;
+
+ FileStatus status = fs.getFileStatus(new org.apache.hadoop.fs.Path(backupPath));
FsPermission perm = status.getPermission();
assertEquals(FsAction.ALL, perm.getUserAction());
- assertEquals(FsAction.ALL, perm.getGroupAction());
- assertEquals(FsAction.ALL, perm.getOtherAction());
+ assertEquals(expectedPerms, perm.getGroupAction());
+ assertEquals(expectedPerms, perm.getOtherAction());
}
}
}
diff --git a/solr/core/src/test/org/apache/solr/handler/TestIncrementalCoreBackup.java b/solr/core/src/test/org/apache/solr/handler/TestIncrementalCoreBackup.java
new file mode 100644
index 00000000000..531b1beb7b0
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/TestIncrementalCoreBackup.java
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler;
+
+import org.apache.lucene.index.IndexCommit;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.params.CoreAdminParams;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.backup.BackupFilePaths;
+import org.apache.solr.core.backup.BackupId;
+import org.apache.solr.core.backup.ShardBackupId;
+import org.apache.solr.core.backup.ShardBackupMetadata;
+import org.apache.solr.core.backup.repository.BackupRepository;
+import org.apache.solr.handler.admin.CoreAdminHandler;
+import org.apache.solr.response.SolrQueryResponse;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Paths;
+import java.util.Arrays;
+
+public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {
+ @Before // unique core per test
+ public void coreInit() throws Exception {
+ initCore("solrconfig.xml", "schema.xml");
+ }
+ @After // unique core per test
+ public void coreDestroy() throws Exception {
+ deleteCore();
+ }
+
+ @Test
+ public void testBackupWithDocsNotSearchable() throws Exception {
+ //See SOLR-11616 to see when this issue can be triggered
+
+ assertU(adoc("id", "1"));
+ assertU(commit());
+
+ assertU(adoc("id", "2"));
+
+ assertU(commit("openSearcher", "false"));
+ assertQ(req("q", "*:*"), "//result[@numFound='1']");
+ assertQ(req("q", "id:1"), "//result[@numFound='1']");
+ assertQ(req("q", "id:2"), "//result[@numFound='0']");
+
+ //call backup
+ final URI location = createAndBootstrapLocationForBackup();
+ final ShardBackupId shardBackupId = new ShardBackupId("shard1", BackupId.zero());
+
+ final CoreContainer cores = h.getCoreContainer();
+ cores.getAllowPaths().add(Paths.get(location));
+ try (final CoreAdminHandler admin = new CoreAdminHandler(cores)) {
+ SolrQueryResponse resp = new SolrQueryResponse();
+ admin.handleRequestBody
+ (req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
+ "core", DEFAULT_TEST_COLLECTION_NAME,
+ "location", location.getPath(),
+ CoreAdminParams.SHARD_BACKUP_ID, shardBackupId.getIdAsString())
+ , resp);
+ assertNull("Backup should have succeeded", resp.getException());
+ simpleBackupCheck(location, shardBackupId);
+ }
+ }
+
+ public void testBackupBeforeFirstCommit() throws Exception {
+
+ // even w/o a user sending any data, the SolrCore initialiation logic should have automatically created
+ // an "empty" commit point that can be backed up...
+ final IndexCommit empty = h.getCore().getDeletionPolicy().getLatestCommit();
+ assertNotNull(empty);
+
+ // white box sanity check that the commit point of the "reader" available from SolrIndexSearcher
+ // matches the commit point that IDPW claims is the "latest"
+ //
+ // this is important to ensure that backup/snapshot behavior is consistent with user expection
+ // when using typical commit + openSearcher
+ assertEquals(empty, h.getCore().withSearcher(s -> s.getIndexReader().getIndexCommit()));
+
+ assertEquals(1L, empty.getGeneration());
+ assertNotNull(empty.getSegmentsFileName());
+ final String initialEmptyIndexSegmentFileName = empty.getSegmentsFileName();
+
+ final CoreContainer cores = h.getCoreContainer();
+ final CoreAdminHandler admin = new CoreAdminHandler(cores);
+ final URI location = createAndBootstrapLocationForBackup();
+
+ final ShardBackupId firstShardBackup = new ShardBackupId("shard1", BackupId.zero());
+ { // first a backup before we've ever done *anything*...
+ SolrQueryResponse resp = new SolrQueryResponse();
+ admin.handleRequestBody
+ (req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
+ "core", DEFAULT_TEST_COLLECTION_NAME,
+ "location", location.getPath(),
+ CoreAdminParams.SHARD_BACKUP_ID, firstShardBackup.getIdAsString()),
+ resp);
+ assertNull("Backup should have succeeded", resp.getException());
+ simpleBackupCheck(location, firstShardBackup, initialEmptyIndexSegmentFileName);
+ }
+
+ { // Empty (named) snapshot..
+ SolrQueryResponse resp = new SolrQueryResponse();
+ admin.handleRequestBody
+ (req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.CREATESNAPSHOT.toString(),
+ "core", DEFAULT_TEST_COLLECTION_NAME,
+ "commitName", "empty_snapshotA"),
+ resp);
+ assertNull("Snapshot A should have succeeded", resp.getException());
+ }
+
+ assertU(adoc("id", "1")); // uncommitted
+
+ final ShardBackupId secondShardBackupId = new ShardBackupId("shard1", new BackupId(1));
+ { // second backup w/uncommited docs
+ SolrQueryResponse resp = new SolrQueryResponse();
+ admin.handleRequestBody
+ (req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
+ "core", DEFAULT_TEST_COLLECTION_NAME,
+ "location", location.getPath(),
+ CoreAdminParams.SHARD_BACKUP_ID, secondShardBackupId.getIdAsString()),
+ resp);
+ assertNull("Backup should have succeeded", resp.getException());
+ simpleBackupCheck(location, secondShardBackupId, initialEmptyIndexSegmentFileName);
+ }
+
+ { // Second empty (named) snapshot..
+ SolrQueryResponse resp = new SolrQueryResponse();
+ admin.handleRequestBody
+ (req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.CREATESNAPSHOT.toString(),
+ "core", DEFAULT_TEST_COLLECTION_NAME,
+ "commitName", "empty_snapshotB"),
+ resp);
+ assertNull("Snapshot A should have succeeded", resp.getException());
+ }
+
+ // Committing the doc now should not affect the existing backups or snapshots...
+ assertU(commit());
+
+ for (ShardBackupId shardBackupId: Arrays.asList(firstShardBackup, secondShardBackupId)) {
+ simpleBackupCheck(location, shardBackupId, initialEmptyIndexSegmentFileName);
+ }
+
+ // Make backups from each of the snapshots and check they are still empty as well...
+ {
+ final ShardBackupId thirdShardBackup = new ShardBackupId("shard1", new BackupId(2));
+ SolrQueryResponse resp = new SolrQueryResponse();
+ admin.handleRequestBody
+ (req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
+ "core", DEFAULT_TEST_COLLECTION_NAME,
+ "commitName", "empty_snapshotA",
+ "location", location.getPath(),
+ CoreAdminParams.SHARD_BACKUP_ID, thirdShardBackup.getIdAsString()),
+ resp);
+ assertNull("Backup from snapshot empty_snapshotA should have succeeded", resp.getException());
+ simpleBackupCheck(location, thirdShardBackup, initialEmptyIndexSegmentFileName);
+ }
+ {
+ final ShardBackupId fourthShardBackup = new ShardBackupId("shard1", new BackupId(3));
+ SolrQueryResponse resp = new SolrQueryResponse();
+ admin.handleRequestBody
+ (req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
+ "core", DEFAULT_TEST_COLLECTION_NAME,
+ "commitName", "empty_snapshotB",
+ "location", location.getPath(),
+ CoreAdminParams.SHARD_BACKUP_ID, fourthShardBackup.getIdAsString()),
+ resp);
+ assertNull("Backup from snapshot empty_snapshotB should have succeeded", resp.getException());
+ simpleBackupCheck(location, fourthShardBackup, initialEmptyIndexSegmentFileName);
+ }
+ admin.close();
+ }
+
+ /**
+ * Tests that a softCommit does not affect what data is in a backup
+ */
+ public void testBackupAfterSoftCommit() throws Exception {
+
+ // sanity check empty index...
+ assertQ(req("q", "id:42"), "//result[@numFound='0']");
+ assertQ(req("q", "id:99"), "//result[@numFound='0']");
+ assertQ(req("q", "*:*"), "//result[@numFound='0']");
+
+ // hard commit one doc...
+ assertU(adoc("id", "99"));
+ assertU(commit());
+ assertQ(req("q", "id:99"), "//result[@numFound='1']");
+ assertQ(req("q", "*:*"), "//result[@numFound='1']");
+
+ final IndexCommit oneDocCommit = h.getCore().getDeletionPolicy().getLatestCommit();
+ assertNotNull(oneDocCommit);
+ final String oneDocSegmentFile = oneDocCommit.getSegmentsFileName();
+
+ final CoreContainer cores = h.getCoreContainer();
+ final CoreAdminHandler admin = new CoreAdminHandler(cores);
+ final URI location = createAndBootstrapLocationForBackup();
+
+ final ShardBackupId firstShardBackupId = new ShardBackupId("shard1", BackupId.zero());
+ { // take an initial 'backup1a' containing our 1 document
+ final SolrQueryResponse resp = new SolrQueryResponse();
+ admin.handleRequestBody
+ (req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
+ "core", DEFAULT_TEST_COLLECTION_NAME,
+ "name", "backup1a",
+ "location", location.getPath(),
+ CoreAdminParams.SHARD_BACKUP_ID, firstShardBackupId.getIdAsString()),
+ resp);
+ assertNull("Backup should have succeeded", resp.getException());
+ simpleBackupCheck(location, firstShardBackupId, oneDocSegmentFile);
+ }
+
+ { // and an initial "snapshot1a' that should eventually match
+ SolrQueryResponse resp = new SolrQueryResponse();
+ admin.handleRequestBody
+ (req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.CREATESNAPSHOT.toString(),
+ "core", DEFAULT_TEST_COLLECTION_NAME,
+ "commitName", "snapshot1a"),
+ resp);
+ assertNull("Snapshot 1A should have succeeded", resp.getException());
+ }
+
+ // now we add our 2nd doc, and make it searchable, but we do *NOT* hard commit it to the index dir...
+ assertU(adoc("id", "42"));
+ assertU(commit("softCommit", "true", "openSearcher", "true"));
+
+ assertQ(req("q", "id:99"), "//result[@numFound='1']");
+ assertQ(req("q", "id:42"), "//result[@numFound='1']");
+ assertQ(req("q", "*:*"), "//result[@numFound='2']");
+
+
+ final ShardBackupId secondShardBackupId = new ShardBackupId("shard1", new BackupId(1));
+ { // we now have an index with two searchable docs, but a new 'backup1b' should still
+ // be identical to the previous backup...
+ final SolrQueryResponse resp = new SolrQueryResponse();
+ admin.handleRequestBody
+ (req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
+ "core", DEFAULT_TEST_COLLECTION_NAME,
+ "location", location.getPath(),
+ CoreAdminParams.SHARD_BACKUP_ID, secondShardBackupId.getIdAsString()),
+ resp);
+ assertNull("Backup should have succeeded", resp.getException());
+ simpleBackupCheck(location, secondShardBackupId, oneDocSegmentFile);
+ }
+
+ { // and a second "snapshot1b' should also still be identical
+ SolrQueryResponse resp = new SolrQueryResponse();
+ admin.handleRequestBody
+ (req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.CREATESNAPSHOT.toString(),
+ "core", DEFAULT_TEST_COLLECTION_NAME,
+ "commitName", "snapshot1b"),
+ resp);
+ assertNull("Snapshot 1B should have succeeded", resp.getException());
+ }
+
+ // Hard Committing the 2nd doc now should not affect the existing backups or snapshots...
+ assertU(commit());
+ simpleBackupCheck(location, firstShardBackupId, oneDocSegmentFile); // backup1a
+ simpleBackupCheck(location, secondShardBackupId, oneDocSegmentFile); // backup1b
+
+ final ShardBackupId thirdShardBackupId = new ShardBackupId("shard1", new BackupId(2));
+ { // But we should be able to confirm both docs appear in a new backup (not based on a previous snapshot)
+ final SolrQueryResponse resp = new SolrQueryResponse();
+ admin.handleRequestBody
+ (req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
+ "core", DEFAULT_TEST_COLLECTION_NAME,
+ "location", location.getPath(),
+ CoreAdminParams.SHARD_BACKUP_ID, thirdShardBackupId.getIdAsString()),
+ resp);
+ assertNull("Backup should have succeeded", resp.getException());
+ // TODO This doesn't actually check that backup has both docs! Can we do better than this without doing a full restore?
+ // Maybe validate the new segments_X file at least to show that it's picked up the latest commit?
+ simpleBackupCheck(location, thirdShardBackupId);
+ }
+
+ // if we go back and create backups from our earlier snapshots they should still only
+ // have 1 expected doc...
+ // Make backups from each of the snapshots and check they are still empty as well...
+ final ShardBackupId fourthShardBackupId = new ShardBackupId("shard1", new BackupId(3));
+ {
+ SolrQueryResponse resp = new SolrQueryResponse();
+ admin.handleRequestBody
+ (req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
+ "core", DEFAULT_TEST_COLLECTION_NAME,
+ "commitName", "snapshot1a",
+ "location", location.getPath(),
+ CoreAdminParams.SHARD_BACKUP_ID, fourthShardBackupId.getIdAsString()),
+ resp);
+ assertNull("Backup of snapshot1a should have succeeded", resp.getException());
+ simpleBackupCheck(location, fourthShardBackupId, oneDocSegmentFile);
+ }
+ final ShardBackupId fifthShardBackupId = new ShardBackupId("shard1", new BackupId(4));
+ {
+ SolrQueryResponse resp = new SolrQueryResponse();
+ admin.handleRequestBody
+ (req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
+ "core", DEFAULT_TEST_COLLECTION_NAME,
+ "commitName", "snapshot1b",
+ "location", location.getPath(),
+ CoreAdminParams.SHARD_BACKUP_ID, fifthShardBackupId.getIdAsString()),
+ resp);
+ assertNull("Backup of snapshot1b should have succeeded", resp.getException());
+ simpleBackupCheck(location, fifthShardBackupId, oneDocSegmentFile);
+ }
+
+ admin.close();
+ }
+
+ /**
+ * Check that the backup metadata file exists, and the corresponding index files can be found.
+ */
+ private static void simpleBackupCheck(URI locationURI, ShardBackupId shardBackupId, String... expectedIndexFiles) throws IOException {
+ try(BackupRepository backupRepository = h.getCoreContainer().newBackupRepository(null)) {
+ final BackupFilePaths backupFilePaths = new BackupFilePaths(backupRepository, locationURI);
+
+ // Ensure that the overall file structure looks correct.
+ assertTrue(backupRepository.exists(locationURI));
+ assertTrue(backupRepository.exists(backupFilePaths.getIndexDir()));
+ assertTrue(backupRepository.exists(backupFilePaths.getShardBackupMetadataDir()));
+ final String metadataFilename = shardBackupId.getBackupMetadataFilename();
+ final URI shardBackupMetadataURI = backupRepository.resolve(backupFilePaths.getShardBackupMetadataDir(), metadataFilename);
+ assertTrue(backupRepository.exists(shardBackupMetadataURI));
+
+ // Ensure that all files listed in the shard-meta file are stored in the index dir
+ final ShardBackupMetadata backupMetadata = ShardBackupMetadata.from(backupRepository,
+ backupFilePaths.getShardBackupMetadataDir(), shardBackupId);
+ for (String indexFileName : backupMetadata.listUniqueFileNames()) {
+ final URI indexFileURI = backupRepository.resolve(backupFilePaths.getIndexDir(), indexFileName);
+ assertTrue("Expected " + indexFileName + " to exist in " + backupFilePaths.getIndexDir(), backupRepository.exists(indexFileURI));
+ }
+
+
+ // Ensure that the expected filenames (if any are provided) exist
+ for (String expectedIndexFile : expectedIndexFiles) {
+ assertTrue("Expected backup to hold a renamed copy of " + expectedIndexFile,
+ backupMetadata.listOriginalFileNames().contains(expectedIndexFile));
+ }
+ }
+ }
+
+ private URI createAndBootstrapLocationForBackup() throws IOException {
+ final File locationFile = createTempDir().toFile();
+ final String location = locationFile.getAbsolutePath();
+
+ h.getCoreContainer().getAllowPaths().add(locationFile.toPath());
+ try (BackupRepository backupRepo = h.getCoreContainer().newBackupRepository(null)) {
+ final BackupFilePaths backupFilePaths = new BackupFilePaths(backupRepo, backupRepo.createURI(location));
+ backupFilePaths.createIncrementalBackupFolders();
+ return backupRepo.createURI(location);
+ }
+ }
+}
+
diff --git a/solr/core/src/test/org/apache/solr/handler/TestCoreBackup.java b/solr/core/src/test/org/apache/solr/handler/TestSnapshotCoreBackup.java
similarity index 93%
rename from solr/core/src/test/org/apache/solr/handler/TestCoreBackup.java
rename to solr/core/src/test/org/apache/solr/handler/TestSnapshotCoreBackup.java
index 858b8072e03..688325d0993 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestCoreBackup.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestSnapshotCoreBackup.java
@@ -34,7 +34,7 @@ import java.io.IOException;
import java.nio.file.Paths;
import java.util.Arrays;
-public class TestCoreBackup extends SolrTestCaseJ4 {
+public class TestSnapshotCoreBackup extends SolrTestCaseJ4 {
@Before // unique core per test
public void coreInit() throws Exception {
initCore("solrconfig.xml", "schema.xml");
@@ -65,10 +65,10 @@ public class TestCoreBackup extends SolrTestCaseJ4 {
cores.getAllowPaths().add(Paths.get(location));
try (final CoreAdminHandler admin = new CoreAdminHandler(cores)) {
SolrQueryResponse resp = new SolrQueryResponse();
- admin.handleRequestBody
- (req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
- "core", DEFAULT_TEST_COLLECTION_NAME, "name", snapshotName, "location", location)
- , resp);
+ admin.handleRequestBody(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
+ "core", DEFAULT_TEST_COLLECTION_NAME, "name", snapshotName, "location",
+ location, CoreAdminParams.BACKUP_INCREMENTAL, "false"),
+ resp);
assertNull("Backup should have succeeded", resp.getException());
simpleBackupCheck(new File(location, "snapshot." + snapshotName), 2);
}
@@ -104,7 +104,8 @@ public class TestCoreBackup extends SolrTestCaseJ4 {
(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
"core", DEFAULT_TEST_COLLECTION_NAME,
"name", "empty_backup1",
- "location", backupDir.getAbsolutePath()),
+ "location", backupDir.getAbsolutePath(),
+ CoreAdminParams.BACKUP_INCREMENTAL, "false"),
resp);
assertNull("Backup should have succeeded", resp.getException());
simpleBackupCheck(new File(backupDir, "snapshot.empty_backup1"),
@@ -129,7 +130,8 @@ public class TestCoreBackup extends SolrTestCaseJ4 {
(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
"core", DEFAULT_TEST_COLLECTION_NAME,
"name", "empty_backup2",
- "location", backupDir.getAbsolutePath()),
+ "location", backupDir.getAbsolutePath(),
+ CoreAdminParams.BACKUP_INCREMENTAL, "false"),
resp);
assertNull("Backup should have succeeded", resp.getException());
simpleBackupCheck(new File(backupDir, "snapshot.empty_backup2"),
@@ -164,7 +166,8 @@ public class TestCoreBackup extends SolrTestCaseJ4 {
"core", DEFAULT_TEST_COLLECTION_NAME,
"name", name,
"commitName", snapName,
- "location", backupDir.getAbsolutePath()),
+ "location", backupDir.getAbsolutePath(),
+ CoreAdminParams.BACKUP_INCREMENTAL, "false"),
resp);
assertNull("Backup "+name+" should have succeeded", resp.getException());
simpleBackupCheck(new File(backupDir, "snapshot." + name),
@@ -207,7 +210,8 @@ public class TestCoreBackup extends SolrTestCaseJ4 {
(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
"core", DEFAULT_TEST_COLLECTION_NAME,
"name", "backup1a",
- "location", backupDir.getAbsolutePath()),
+ "location", backupDir.getAbsolutePath(),
+ CoreAdminParams.BACKUP_INCREMENTAL, "false"),
resp);
assertNull("Backup should have succeeded", resp.getException());
simpleBackupCheck(new File(backupDir, "snapshot.backup1a"),
@@ -240,7 +244,8 @@ public class TestCoreBackup extends SolrTestCaseJ4 {
(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
"core", DEFAULT_TEST_COLLECTION_NAME,
"name", "backup1b",
- "location", backupDir.getAbsolutePath()),
+ "location", backupDir.getAbsolutePath(),
+ CoreAdminParams.BACKUP_INCREMENTAL, "false"),
resp);
assertNull("Backup should have succeeded", resp.getException());
simpleBackupCheck(new File(backupDir, "snapshot.backup1b"),
@@ -271,7 +276,8 @@ public class TestCoreBackup extends SolrTestCaseJ4 {
(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
"core", DEFAULT_TEST_COLLECTION_NAME,
"name", "backup2",
- "location", backupDir.getAbsolutePath()),
+ "location", backupDir.getAbsolutePath(),
+ CoreAdminParams.BACKUP_INCREMENTAL, "false"),
resp);
assertNull("Backup should have succeeded", resp.getException());
simpleBackupCheck(new File(backupDir, "snapshot.backup2"), 2);
@@ -288,7 +294,8 @@ public class TestCoreBackup extends SolrTestCaseJ4 {
"core", DEFAULT_TEST_COLLECTION_NAME,
"name", name,
"commitName", snapName,
- "location", backupDir.getAbsolutePath()),
+ "location", backupDir.getAbsolutePath(),
+ CoreAdminParams.BACKUP_INCREMENTAL, "false"),
resp);
assertNull("Backup "+name+" should have succeeded", resp.getException());
simpleBackupCheck(new File(backupDir, "snapshot." + name),
diff --git a/solr/core/src/test/org/apache/solr/handler/TestStressIncrementalBackup.java b/solr/core/src/test/org/apache/solr/handler/TestStressIncrementalBackup.java
new file mode 100644
index 00000000000..871d3498be6
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/TestStressIncrementalBackup.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler;
+
+import java.io.File;
+import java.lang.invoke.MethodHandles;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.client.solrj.response.RequestStatusState;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.params.UpdateParams;
+import org.apache.solr.util.LogLevel;
+import org.junit.After;
+import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.handler.TestStressThreadBackup.makeDoc;
+
+//@LuceneTestCase.Nightly
[email protected]({"SimpleText"})
+@LogLevel("org.apache.solr.handler.SnapShooter=DEBUG;org.apache.solr.core.IndexDeletionPolicyWrapper=DEBUG")
+public class TestStressIncrementalBackup extends SolrCloudTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private File backupDir;
+ private SolrClient adminClient;
+ private SolrClient coreClient;
+ @Before
+ public void beforeTest() throws Exception {
+ backupDir = createTempDir(getTestClass().getSimpleName() + "_backups").toFile();
+
+ // NOTE: we don't actually care about using SolrCloud, but we want to use SolrClient and I can't
+ // bring myself to deal with the nonsense that is SolrJettyTestBase.
+
+ // We do however explicitly want a fresh "cluster" every time a test is run
+ configureCluster(1)
+ .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
+ .configure();
+
+ assertEquals(0, (CollectionAdminRequest.createCollection(DEFAULT_TEST_COLLECTION_NAME, "conf1", 1, 1)
+ .process(cluster.getSolrClient()).getStatus()));
+ adminClient = getHttpSolrClient(cluster.getJettySolrRunners().get(0).getBaseUrl().toString());
+ initCoreNameAndSolrCoreClient();
+ }
+
+ private void initCoreNameAndSolrCoreClient() {
+ // Sigh.
+ Replica r = cluster.getSolrClient().getZkStateReader().getClusterState()
+ .getCollection(DEFAULT_TEST_COLLECTION_NAME).getActiveSlices().iterator().next()
+ .getReplicas().iterator().next();
+ coreName = r.getCoreName();
+ coreClient = getHttpSolrClient(r.getCoreUrl());
+ }
+
+ @After
+ public void afterTest() throws Exception {
+ // we use a clean cluster instance for every test, so we need to clean it up
+ shutdownCluster();
+
+ if (null != adminClient) {
+ adminClient.close();
+ }
+ if (null != coreClient) {
+ coreClient.close();
+ }
+ }
+
+ public void testCoreAdminHandler() throws Exception {
+ final int numBackupIters = 20; // don't use 'atLeast', we don't want to blow up on nightly
+
+ final AtomicReference<Throwable> heavyCommitFailure = new AtomicReference<>();
+ final AtomicBoolean keepGoing = new AtomicBoolean(true);
+
+ // this thread will do nothing but add/commit new 'dummy' docs over and over again as fast as possible
+ // to create a lot of index churn w/ segment merging
+ final Thread heavyCommitting = new Thread() {
+ public void run() {
+ try {
+ int docIdCounter = 0;
+ while (keepGoing.get()) {
+ docIdCounter++;
+
+ final UpdateRequest req = new UpdateRequest().add(makeDoc("dummy_" + docIdCounter, "dummy"));
+ // always commit to force lots of new segments
+ req.setParam(UpdateParams.COMMIT,"true");
+ req.setParam(UpdateParams.OPEN_SEARCHER,"false"); // we don't care about searching
+
+ // frequently forceMerge to ensure segments are frequently deleted
+ if (0 == (docIdCounter % 13)) { // arbitrary
+ req.setParam(UpdateParams.OPTIMIZE, "true");
+ req.setParam(UpdateParams.MAX_OPTIMIZE_SEGMENTS, "5"); // arbitrary
+ }
+
+ log.info("Heavy Committing #{}: {}", docIdCounter, req);
+ final UpdateResponse rsp = req.process(coreClient);
+ assertEquals("Dummy Doc#" + docIdCounter + " add status: " + rsp.toString(), 0, rsp.getStatus());
+
+ }
+ } catch (Throwable t) {
+ heavyCommitFailure.set(t);
+ }
+ }
+ };
+
+ heavyCommitting.start();
+ try {
+ // now have the "main" test thread try to take a serious of backups/snapshots
+ // while adding other "real" docs
+
+ // NOTE #1: start at i=1 for 'id' & doc counting purposes...
+ // NOTE #2: abort quickly if the oher thread reports a heavyCommitFailure...
+ for (int i = 1; (i <= numBackupIters && null == heavyCommitFailure.get()); i++) {
+
+ // in each iteration '#i', the commit we create should have exactly 'i' documents in
+ // it with the term 'type_s:real' (regardless of what the other thread does with dummy docs)
+
+ // add & commit a doc #i
+ final UpdateRequest req = new UpdateRequest().add(makeDoc("doc_" + i, "real"));
+ req.setParam(UpdateParams.COMMIT,"true"); // make immediately available for backup
+ req.setParam(UpdateParams.OPEN_SEARCHER,"false"); // we don't care about searching
+
+ final UpdateResponse rsp = req.process(coreClient);
+ assertEquals("Real Doc#" + i + " add status: " + rsp.toString(), 0, rsp.getStatus());
+
+ makeBackup();
+ }
+
+ } finally {
+ keepGoing.set(false);
+ heavyCommitting.join();
+ }
+ assertNull(heavyCommitFailure.get());
+ }
+
+ public void makeBackup() throws Exception {
+ CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(DEFAULT_TEST_COLLECTION_NAME, "stressBackup")
+ .setLocation(backupDir.getAbsolutePath())
+ .setIncremental(true)
+ .setMaxNumberBackupPoints(5);
+ if (random().nextBoolean()) {
+ try {
+ RequestStatusState state = backup.processAndWait(cluster.getSolrClient(), 1000);
+ assertEquals(RequestStatusState.COMPLETED, state);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ } else {
+ CollectionAdminResponse rsp = backup.process(cluster.getSolrClient());
+ assertEquals(0, rsp.getStatus());
+ }
+ }
+
+}
diff --git a/solr/core/src/test/org/apache/solr/handler/TestStressThreadBackup.java b/solr/core/src/test/org/apache/solr/handler/TestStressThreadBackup.java
index b1cc214b130..8c7128bdaf8 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestStressThreadBackup.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestStressThreadBackup.java
@@ -16,20 +16,6 @@
*/
package org.apache.solr.handler;
-import java.io.File;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Queue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
@@ -57,9 +43,24 @@ import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
@Nightly
@SuppressCodecs({"SimpleText"})
@LogLevel("org.apache.solr.handler.SnapShooter=DEBUG;org.apache.solr.core.IndexDeletionPolicyWrapper=DEBUG")
@@ -114,11 +115,13 @@ public class TestStressThreadBackup extends SolrCloudTestCase {
}
}
+ @Test
public void testCoreAdminHandler() throws Exception {
// Use default BackupAPIImpl which hits CoreAdmin API for everything
testSnapshotsAndBackupsDuringConcurrentCommitsAndOptimizes(new BackupAPIImpl());
}
-
+
+ @Test
public void testReplicationHandler() throws Exception {
// Create a custom BackupAPIImpl which uses ReplicatoinHandler for the backups
// but still defaults to CoreAdmin for making named snapshots (since that's what's documented)
@@ -329,7 +332,7 @@ public class TestStressThreadBackup extends SolrCloudTestCase {
* @param id the uniqueKey
* @param type the type of the doc for use in the 'type_s' field (for term counting later)
*/
- private static SolrInputDocument makeDoc(String id, String type) {
+ static SolrInputDocument makeDoc(String id, String type) {
final SolrInputDocument doc = new SolrInputDocument("id", id, "type_s", type);
for (int f = 0; f < 100; f++) {
doc.addField(f + "_s", TestUtil.randomUnicodeString(random(), 20));
diff --git a/solr/solr-ref-guide/src/collection-management.adoc b/solr/solr-ref-guide/src/collection-management.adoc
index 3cfc1d8c1bf..d846c2aab29 100644
--- a/solr/solr-ref-guide/src/collection-management.adoc
+++ b/solr/solr-ref-guide/src/collection-management.adoc
@@ -1211,6 +1211,17 @@ Backs up Solr collections and associated configurations to a shared filesystem -
The BACKUP command will backup Solr indexes and configurations for a specified collection. The BACKUP command <<making-and-restoring-backups.adoc#,takes one copy from each shard for the indexes>>. For configurations, it backs up the configset that was associated with the collection and metadata.
+Backup data is stored in the repository based on the provided `name` and `location`.
+Each backup location can hold multiple backups for the same collection, allowing users to later restore from any of these "backup points" as desired.
+Within a location backups are done incrementally, so that index files uploaded previously are skipped and not duplicated in the backup repository.
+
+[NOTE]
+====
+Previous versions of Solr supported a different snapshot-based backup method without the incremental support described above.
+Solr can still restore from backups that use this old format, but creating new backups of this format is not recommended and snapshot-based backups are officially deprecated.
+See the `incremental` parameter below for more information.
+====
+
=== BACKUP Parameters
`collection`::
@@ -1220,10 +1231,12 @@ The name of the collection to be backed up. This parameter is required.
What to name the backup that is created. This is checked to make sure it doesn't already exist, and otherwise an error message is raised. This parameter is required.
`location`::
-The location on a shared drive for the backup command to write to. Alternately it can be set as a <<cluster-node-management.adoc#clusterprop,cluster property>>.
+The location on a shared drive for the backup command to write to. This parameter is required, unless a default location is defined on the repository configuration, or set as a <<cluster-node-management.adoc#clusterprop,cluster property>>.
+
If the location path is on a mounted drive, the mount must be available on the node that serves as the overseer, even if the overseer node does not host a replica of the collection being backed up.
Since any node can take the overseer role at any time, a best practice to avoid possible backup failures is to ensure the mount point is available on all nodes of the cluster.
++
+Each backup location can only hold a backup for one collection, however the same location can be used for repeated backups of the same collection. Repeated backups of the same collection are done incrementally, so that files unchanged since the last backup are not duplicated in the backup repository.
`async`::
Request ID to track this action which will be <<collections-api.adoc#asynchronous-calls,processed asynchronously>>.
@@ -1231,6 +1244,16 @@ Request ID to track this action which will be <<collections-api.adoc#asynchronou
`repository`::
The name of a repository to be used for the backup. If no repository is specified then the local filesystem repository will be used automatically.
+`maxNumBackupPoints`::
+The upper-bound on how many backups should be retained at the backup location.
+If the current number exceeds this bound, older backups will be deleted until only `maxNumBackupPoints` backups remain.
+This parameter has no effect if `incremental=false` is specified.
+
+`incremental`::
+A boolean parameter allowing users to choose whether to create an incremental (`incremental=true`) or a "snapshot" (`incremental=false`) backup.
+If unspecified, backups are done incrementally by default.
+Incremental backups are preferred in all known circumstances and snapshot backups are deprecated, so this parameter should only be used after much consideration.
+
[[restore]]
== RESTORE: Restore Collection
@@ -1263,6 +1286,11 @@ Request ID to track this action which will be <<collections-api.adoc#asynchronou
`repository`::
The name of a repository to be used for the backup. If no repository is specified then the local filesystem repository will be used automatically.
+`backupId`::
+The ID of a specific backup point to restore from.
++
+Backup locations can hold multiple backups of the same collection. This parameter allows users to choose which of those backups should be used to restore from. If not specified the most recent backup point is used.
+
There are also optional parameters that determine the target collection layout.
The following parameters are currently supported (described in detail in the <<create,CREATE collection>> section):
`createNodeSet`, `createNodeSet.shuffle`.
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index 51349ad841c..e09ebb35a6c 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -983,6 +983,8 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
protected String location;
protected Optional<String> commitName = Optional.empty();
protected Optional<String> indexBackupStrategy = Optional.empty();
+ protected boolean incremental = true;
+ protected Optional<Integer> maxNumBackupPoints = Optional.empty();
public Backup(String collection, String name) {
super(CollectionAction.BACKUP, collection);
@@ -1026,6 +1028,38 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
return this;
}
+ /**
+ * Specifies the backup method to use: the deprecated 'full-snapshot' format, or the current 'incremental' format.
+ *
+ * Defaults to 'true' if unspecified.
+ *
+ * Incremental backups are almost always preferable to the deprecated 'full-snapshot' format, as incremental backups
+ * can take advantage of previously backed-up files and will only upload those that aren't already stored in the
+ * repository - saving lots of time and network bandwidth. The older 'full-snapshot' format should only be used by
+ * experts with a particular reason to do so.
+ *
+ * @param incremental true to use incremental backups, false otherwise.
+ */
+ @Deprecated
+ public Backup setIncremental(boolean incremental) {
+ this.incremental = incremental;
+ return this;
+ }
+
+ /**
+ * Specifies the maximum number of backup points to keep at the backup location.
+ *
+ * If the current backup causes the number of stored backup points to exceed this value, the oldest backup points
+ * are cleaned up so that only {@code #maxNumBackupPoints} are retained.
+ *
+ * This parameter is ignored if the request uses a non-incremental backup.
+ * @param maxNumBackupPoints the number of backup points to retain after the current backup
+ */
+ public Backup setMaxNumberBackupPoints(int maxNumBackupPoints) {
+ this.maxNumBackupPoints = Optional.of(maxNumBackupPoints);
+ return this;
+ }
+
@Override
public SolrParams getParams() {
ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
@@ -1041,6 +1075,10 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
if (indexBackupStrategy.isPresent()) {
params.set(CollectionAdminParams.INDEX_BACKUP_STRATEGY, indexBackupStrategy.get());
}
+ if (maxNumBackupPoints.isPresent()) {
+ params.set(CoreAdminParams.MAX_NUM_BACKUP_POINTS, maxNumBackupPoints.get());
+ }
+ params.set(CoreAdminParams.BACKUP_INCREMENTAL, incremental);
return params;
}
@@ -1065,6 +1103,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
protected Optional<String> createNodeSet = Optional.empty();
protected Optional<Boolean> createNodeSetShuffle = Optional.empty();
protected Properties properties;
+ protected Integer backupId;
public Restore(String collection, String backupName) {
super(CollectionAction.RESTORE, collection);
@@ -1126,6 +1165,21 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
}
public Restore setProperties(Properties properties) { this.properties = properties; return this;}
+ /**
+ * Specify the ID of the backup-point to restore from.
+ *
+ * '-1'q is used by default to have Solr restore from the most recent backup-point.
+ *
+ * Solr can store multiple backup points for a given collection - each identified by a unique backup ID. Users who
+ * want to restore a particular backup-point can specify it using this method.
+ *
+ * @param backupId the ID of the backup-point to restore from
+ */
+ public Restore setBackupId(int backupId) {
+ this.backupId = backupId;
+ return this;
+ }
+
// TODO support rule, snitch
@Override
@@ -1163,6 +1217,9 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
if (createNodeSetShuffle.isPresent()) {
params.set(CREATE_NODE_SET_SHUFFLE_PARAM, createNodeSetShuffle.get());
}
+ if (backupId != null) {
+ params.set(CoreAdminParams.BACKUP_ID, backupId);
+ }
return params;
}
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
index f55cb70e2f5..1ffa05e7980 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
@@ -113,6 +113,8 @@ public interface CollectionParams {
MODIFYCOLLECTION(true, LockLevel.COLLECTION),
BACKUP(true, LockLevel.COLLECTION),
RESTORE(true, LockLevel.COLLECTION),
+ LISTBACKUP(false, LockLevel.NONE),
+ DELETEBACKUP(true, LockLevel.COLLECTION),
CREATESNAPSHOT(true, LockLevel.COLLECTION),
DELETESNAPSHOT(true, LockLevel.COLLECTION),
LISTSNAPSHOTS(false, LockLevel.NONE),
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
index 2548e621315..291ff94b2ef 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
@@ -122,6 +122,31 @@ public abstract class CoreAdminParams
*/
public static final String BACKUP_LOCATION = "location";
+ /**
+ * The ID of the shard-backup prior to the latest one (indicated by {@link #SHARD_BACKUP_ID}
+ */
+ public static final String PREV_SHARD_BACKUP_ID = "prevShardBackupId";
+
+ /**
+ * The ID of the shard-backup in question
+ */
+ public static final String SHARD_BACKUP_ID = "shardBackupId";
+
+ /**
+ * A parameter to specify last number of backups (delete the rest)
+ */
+ public static final String MAX_NUM_BACKUP_POINTS = "maxNumBackupPoints";
+
+ /**
+ * Unique id of the backup
+ */
+ public static final String BACKUP_ID = "backupId";
+
+ /**
+ * A parameter to specify whether incremental backup is used
+ */
+ public static final String BACKUP_INCREMENTAL = "incremental";
+
/**
* A parameter to specify the name of the commit to be stored during the backup operation.
*/
diff --git a/solr/solrj/src/resources/apispec/collections.Commands.json b/solr/solrj/src/resources/apispec/collections.Commands.json
index 747cc8b8b28..4d4a1784fcb 100644
--- a/solr/solrj/src/resources/apispec/collections.Commands.json
+++ b/solr/solrj/src/resources/apispec/collections.Commands.json
@@ -245,6 +245,14 @@
"type": "string",
"description": "A location on a shared drive for the backup-collection command to write to. Alternately, it can be set as a cluster property with the cluster endpoint, which also supports setting a location."
},
+ "followAliases": {
+ "type": "boolean",
+ "description": "Controls whether aliases are resolved when trying to back up the specified collection, or whether Solr should only backup the provided collection name if it matches a concrete collection."
+ },
+ "incremental": {
+ "type": "boolean",
+ "description": "An internal property that controls whether the backup should use the standard 'incremental' file format or the deprecated 'full-snapshot' based format."
+ },
"async": {
"type": "string",
"description": "Defines a request ID that can be used to track this action after it's submitted. The action will be processed asynchronously."
@@ -252,8 +260,7 @@
},
"required": [
"collection",
- "name",
- "location"
+ "name"
]
},
"restore-collection": {
@@ -273,6 +280,10 @@
"type": "string",
"description": "The location on the shared drive for the restore-collection command to read from. Alternately, it can be set as a cluster property with the cluster endpoint, which also supports setting a location."
},
+ "backupId": {
+ "type": "integer",
+ "description": "The ID of the backup to restore, when the provided location and backup name hold multiple backups for the provided collection. Defaults to the most recent backup if not specified."
+ },
"async": {
"type": "string",
"description": "Defines a request ID that can be used to track this action after it's submitted. The action will be processed asynchronously."
@@ -280,8 +291,7 @@
},
"required": [
"collection",
- "name",
- "location"
+ "name"
]
}
}
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractBackupRepositoryTest.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractBackupRepositoryTest.java
new file mode 100644
index 00000000000..eb46d778adc
--- /dev/null
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractBackupRepositoryTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.api.collections;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.NoSuchFileException;
+import java.util.Arrays;
+import java.util.HashSet;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.core.backup.repository.BackupRepository;
+import org.junit.Test;
+
+public abstract class AbstractBackupRepositoryTest extends SolrTestCaseJ4 {
+
+ protected abstract BackupRepository getRepository();
+
+ protected abstract URI getBaseUri() throws URISyntaxException;
+
+ @Test
+ public void test() throws IOException, URISyntaxException {
+ try (BackupRepository repo = getRepository()) {
+ URI baseUri = repo.resolve(getBaseUri(), "tmp");
+ if (repo.exists(baseUri)) {
+ repo.deleteDirectory(baseUri);
+ }
+ assertFalse(repo.exists(baseUri));
+ repo.createDirectory(baseUri);
+ assertTrue(repo.exists(baseUri));
+ assertEquals(0, repo.listAll(baseUri).length);
+
+ // test nested structure
+ URI tmpFolder = repo.resolve(baseUri, "tmpDir");
+ repo.createDirectory(tmpFolder);
+ assertEquals(repo.getPathType(tmpFolder), BackupRepository.PathType.DIRECTORY);
+ addFile(repo, repo.resolve(tmpFolder, "file1"));
+ addFile(repo, repo.resolve(tmpFolder, "file2"));
+ assertEquals(repo.getPathType(repo.resolve(tmpFolder, "file1")), BackupRepository.PathType.FILE);
+ String[] files = repo.listAll(tmpFolder);
+ assertEquals(new HashSet<>(Arrays.asList("file1", "file2")), new HashSet<>(Arrays.asList(files)));
+
+ URI tmpFolder2 = repo.resolve(tmpFolder, "tmpDir2");
+ repo.createDirectory(tmpFolder2);
+ addFile(repo, repo.resolve(tmpFolder2, "file3"));
+ addFile(repo, repo.resolve(tmpFolder2, "file4"));
+ addFile(repo, repo.resolve(tmpFolder2, "file5"));
+ //2 files + 1 folder
+ assertEquals(3, repo.listAll(tmpFolder).length);
+ // create same directory must be a no-op
+ repo.createDirectory(tmpFolder2);
+ assertEquals(3, repo.listAll(tmpFolder2).length);
+ assertTrue(repo.exists(tmpFolder2));
+ assertTrue(repo.exists(repo.resolve(tmpFolder2, "file3")));
+ try {
+ repo.delete(tmpFolder2, Arrays.asList("file7", "file6"), false);
+ fail("Delete non existence file leads to success");
+ } catch (NoSuchFileException e) {
+ // expected
+ }
+ repo.delete(tmpFolder2, Arrays.asList("file7", "file6"), true);
+ repo.delete(tmpFolder2, Arrays.asList("file3", "file4"), true);
+ assertEquals(1, repo.listAll(tmpFolder2).length);
+ assertFalse(repo.exists(repo.resolve(tmpFolder2, "file3")));
+ repo.deleteDirectory(tmpFolder);
+ assertFalse(repo.exists(tmpFolder));
+ assertFalse(repo.exists(repo.resolve(tmpFolder2, "file5")));
+ assertFalse(repo.exists(repo.resolve(tmpFolder, "file1")));
+ }
+ }
+
+ private void addFile(BackupRepository repo, URI file) throws IOException {
+ try (OutputStream os = repo.createOutput(file)) {
+ os.write(100);
+ os.write(101);
+ os.write(102);
+ }
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java
similarity index 96%
rename from solr/core/src/test/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java
rename to solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java
index ccfc78dc9ee..74266aa745c 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java
@@ -54,7 +54,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * This class implements the logic required to test Solr cloud backup/restore capability.
+ * Used to test the traditional (now deprecated) 'full-snapshot' method of backup/restoration.
+ *
+ * For a similar test harness for incremental backup/restoration see {@link AbstractIncrementalBackupTest}
*/
public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -182,7 +184,9 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa
{
CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(getCollectionName(), backupName)
- .setLocation(backupLocation).setRepositoryName(getBackupRepoName());
+ .setLocation(backupLocation)
+ .setIncremental(false)
+ .setRepositoryName(getBackupRepoName());
assertEquals(0, backup.process(solrClient).getStatus());
}
@@ -192,7 +196,8 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa
{
CollectionAdminRequest.Restore restore = CollectionAdminRequest.restoreCollection(restoreCollectionName, backupName)
- .setLocation(backupLocation).setRepositoryName(getBackupRepoName());
+ .setLocation(backupLocation)
+ .setRepositoryName(getBackupRepoName());
restore.setConfigName("confFaulty");
assertEquals(RequestStatusState.FAILED, restore.processAndWait(solrClient, 30));
@@ -231,6 +236,7 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa
// Do not specify the backup location.
CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(collectionName, backupName)
+ .setIncremental(false)
.setRepositoryName(getBackupRepoName());
try {
backup.process(solrClient);
@@ -293,7 +299,9 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa
{
CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(collectionName, backupName)
- .setLocation(backupLocation).setRepositoryName(getBackupRepoName());
+ .setIncremental(false)
+ .setLocation(backupLocation)
+ .setRepositoryName(getBackupRepoName());
if (random().nextBoolean()) {
assertEquals(0, backup.process(client).getStatus());
} else {
@@ -400,7 +408,7 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa
// TODO Find the applicable core.properties on the file system but how?
}
- private Map<String, Integer> getShardToDocCountMap(CloudSolrClient client, DocCollection docCollection) throws SolrServerException, IOException {
+ public static Map<String, Integer> getShardToDocCountMap(CloudSolrClient client, DocCollection docCollection) throws SolrServerException, IOException {
Map<String,Integer> shardToDocCount = new TreeMap<>();
for (Slice slice : docCollection.getActiveSlices()) {
String shardName = slice.getName();
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java
new file mode 100644
index 00000000000..3218bbf93e1
--- /dev/null
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java
@@ -0,0 +1,487 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.api.collections;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.index.IndexCommit;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.TestUtil;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.client.solrj.response.RequestStatusState;
+import org.apache.solr.cloud.AbstractDistribZkTestBase;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.DirectoryFactory;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.TrackingBackupRepository;
+import org.apache.solr.core.backup.BackupFilePaths;
+import org.apache.solr.core.backup.BackupId;
+import org.apache.solr.core.backup.BackupProperties;
+import org.apache.solr.core.backup.Checksum;
+import org.apache.solr.core.backup.ShardBackupId;
+import org.apache.solr.core.backup.ShardBackupMetadata;
+import org.apache.solr.core.backup.repository.BackupRepository;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.apache.solr.core.TrackingBackupRepository.copiedFiles;
+
+/**
+ * Used to test the incremental method of backup/restoration (as opposed to the deprecated 'full snapshot' method).
+ *
+ * For a similar test harness for snapshot backup/restoration see {@link AbstractCloudBackupRestoreTestCase}
+ */
+public abstract class AbstractIncrementalBackupTest extends SolrCloudTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static long docsSeed; // see indexDocs()
+ protected static final int NUM_SHARDS = 2;//granted we sometimes shard split to get more
+ protected static final String BACKUPNAME_PREFIX = "mytestbackup";
+ protected static final String BACKUP_REPO_NAME = "trackingBackupRepository";
+
+ protected String testSuffix = "test1";
+ protected int replFactor;
+ protected int numTlogReplicas;
+ protected int numPullReplicas;
+
+ @BeforeClass
+ public static void createCluster() throws Exception {
+ docsSeed = random().nextLong();
+ System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
+ }
+
+ /**
+ * @return The name of the collection to use.
+ */
+ public abstract String getCollectionNamePrefix();
+
+ public String getCollectionName(){
+ return getCollectionNamePrefix() + "_" + testSuffix;
+ }
+
+ public void setTestSuffix(String testSuffix) {
+ this.testSuffix = testSuffix;
+ }
+
+ private void randomizeReplicaTypes() {
+ replFactor = TestUtil.nextInt(random(), 1, 2);
+// numTlogReplicas = TestUtil.nextInt(random(), 0, 1);
+// numPullReplicas = TestUtil.nextInt(random(), 0, 1);
+ }
+
+ /**
+ * @return The absolute path for the backup location.
+ * Could return null.
+ */
+ public abstract String getBackupLocation();
+
+ @Test
+ public void testSimple() throws Exception {
+ final String backupCollectionName = getCollectionName();
+ final String restoreCollectionName = backupCollectionName + "_restore";
+ TrackingBackupRepository.clear();
+
+ setTestSuffix("testbackupincsimple");
+ CloudSolrClient solrClient = cluster.getSolrClient();
+
+ CollectionAdminRequest
+ .createCollection(backupCollectionName, "conf1", NUM_SHARDS, 1)
+ .process(solrClient);
+ int expectedNumDocs = indexDocs(backupCollectionName, true);
+ String backupName = BACKUPNAME_PREFIX + testSuffix;
+ try (BackupRepository repository = cluster.getJettySolrRunner(0).getCoreContainer()
+ .newBackupRepository(BACKUP_REPO_NAME)) {
+ String backupLocation = repository.getBackupLocation(getBackupLocation());
+ long t = System.nanoTime();
+ CollectionAdminRequest.backupCollection(backupCollectionName, backupName)
+ .setLocation(backupLocation)
+ .setIncremental(true)
+ .setRepositoryName(BACKUP_REPO_NAME)
+ .processAndWait(cluster.getSolrClient(), 100);
+ long timeTaken = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - t);
+ log.info("Created backup with {} docs, took {}ms", expectedNumDocs, timeTaken);
+ expectedNumDocs += indexDocs(backupCollectionName, true);
+
+ t = System.nanoTime();
+ CollectionAdminRequest.backupCollection(backupCollectionName, backupName)
+ .setLocation(backupLocation)
+ .setIncremental(true)
+ .setRepositoryName(BACKUP_REPO_NAME)
+ .processAndWait(cluster.getSolrClient(), 100);
+ timeTaken = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - t);
+ long numFound = cluster.getSolrClient().query(backupCollectionName,
+ new SolrQuery("*:*")).getResults().getNumFound();
+ log.info("Created backup with {} docs, took {}ms", numFound, timeTaken);
+
+ t = System.nanoTime();
+ CollectionAdminRequest.restoreCollection(restoreCollectionName, backupName)
+ .setBackupId(0)
+ .setLocation(backupLocation).setRepositoryName(BACKUP_REPO_NAME).processAndWait(solrClient, 500);
+ timeTaken = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - t);
+ log.info("Restored from backup, took {}ms", timeTaken);
+ numFound = cluster.getSolrClient().query(restoreCollectionName,
+ new SolrQuery("*:*")).getResults().getNumFound();
+ assertEquals(expectedNumDocs, numFound);
+ }
+ }
+
+ @Test
+ @Slow
+ @SuppressWarnings("rawtypes")
+ public void testBackupIncremental() throws Exception {
+ TrackingBackupRepository.clear();
+
+ setTestSuffix("testbackupinc");
+ randomizeReplicaTypes();
+ CloudSolrClient solrClient = cluster.getSolrClient();
+
+ CollectionAdminRequest
+ .createCollection(getCollectionName(), "conf1", NUM_SHARDS, replFactor, numTlogReplicas, numPullReplicas)
+ .process(solrClient);
+
+ indexDocs(getCollectionName(), false);
+
+ String backupName = BACKUPNAME_PREFIX + testSuffix;
+ try (BackupRepository repository = cluster.getJettySolrRunner(0).getCoreContainer()
+ .newBackupRepository(BACKUP_REPO_NAME)) {
+ String backupLocation = repository.getBackupLocation(getBackupLocation());
+ URI uri = repository.resolve(repository.createURI(backupLocation), backupName);
+ BackupFilePaths backupPaths = new BackupFilePaths(repository, uri);
+ IncrementalBackupVerifier verifier = new IncrementalBackupVerifier(repository, backupLocation, backupName, getCollectionName(), 3);
+
+ backupRestoreThenCheck(solrClient, verifier);
+ indexDocs(getCollectionName(), false);
+ backupRestoreThenCheck(solrClient, verifier);
+
+ // adding more commits to trigger merging segments
+ for (int i = 0; i < 15; i++) {
+ indexDocs(getCollectionName(), 5,false);
+ }
+ backupRestoreThenCheck(solrClient, verifier);
+
+ indexDocs(getCollectionName(), false);
+ backupRestoreThenCheck(solrClient, verifier);
+
+ simpleRestoreAndCheckDocCount(solrClient, backupLocation, backupName);
+
+ new UpdateRequest()
+ .deleteByQuery("*:*")
+ .commit(cluster.getSolrClient(), getCollectionName());
+ indexDocs(getCollectionName(), false);
+ // corrupt index files
+ corruptIndexFiles();
+ try {
+ log.info("Create backup after corrupt index files");
+ CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(getCollectionName(), backupName)
+ .setLocation(backupLocation)
+ .setIncremental(true)
+ .setMaxNumberBackupPoints(3)
+ .setRepositoryName(BACKUP_REPO_NAME);
+ if (random().nextBoolean()) {
+ RequestStatusState state = backup.processAndWait(cluster.getSolrClient(), 1000);
+ if (state != RequestStatusState.FAILED) {
+ fail("This backup should be failed");
+ }
+ } else {
+ CollectionAdminResponse rsp = backup.process(cluster.getSolrClient());
+ fail("This backup should be failed");
+ }
+ } catch (Exception e) {
+ // expected
+ e.printStackTrace();
+ }
+ }
+ }
+
+ protected void corruptIndexFiles() throws IOException {
+ Collection<Slice> slices = getCollectionState(getCollectionName()).getSlices();
+ Slice slice = slices.iterator().next();
+ JettySolrRunner leaderNode = cluster.getReplicaJetty(slice.getLeader());
+
+ SolrCore solrCore = leaderNode.getCoreContainer().getCore(slice.getLeader().getCoreName());
+ Set<String> fileNames = new HashSet<>(solrCore.getDeletionPolicy().getLatestCommit().getFileNames());
+ File indexFolder = new File(solrCore.getIndexDir());
+ File fileGetCorrupted = Stream.of(Objects.requireNonNull(indexFolder.listFiles()))
+ .filter(x -> fileNames.contains(x.getName()))
+ .findAny().get();
+ try (FileInputStream fis = new FileInputStream(fileGetCorrupted)){
+ byte[] contents = IOUtils.readFully(fis, (int) fileGetCorrupted.length());
+ contents[contents.length - CodecUtil.footerLength() - 1] += 1;
+ contents[contents.length - CodecUtil.footerLength() - 2] += 1;
+ contents[contents.length - CodecUtil.footerLength() - 3] += 1;
+ contents[contents.length - CodecUtil.footerLength() - 4] += 1;
+ try (FileOutputStream fos = new FileOutputStream(fileGetCorrupted)) {
+ IOUtils.write(contents, fos);
+ }
+ } finally {
+ solrCore.close();
+ }
+ }
+
+ private void backupRestoreThenCheck(CloudSolrClient solrClient,
+ IncrementalBackupVerifier verifier) throws Exception {
+ verifier.incrementalBackupThenVerify();
+
+ if( random().nextBoolean() )
+ simpleRestoreAndCheckDocCount(solrClient, verifier.backupLocation, verifier.backupName);
+ }
+
+ private void simpleRestoreAndCheckDocCount(CloudSolrClient solrClient, String backupLocation, String backupName) throws Exception{
+ Map<String, Integer> origShardToDocCount = AbstractCloudBackupRestoreTestCase.getShardToDocCountMap(solrClient, getCollectionState(getCollectionName()));
+
+ String restoreCollectionName = getCollectionName() + "_restored";
+
+ CollectionAdminRequest.restoreCollection(restoreCollectionName, backupName)
+ .setLocation(backupLocation).setRepositoryName(BACKUP_REPO_NAME).process(solrClient);
+
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(
+ restoreCollectionName, cluster.getSolrClient().getZkStateReader(), log.isDebugEnabled(), true, 30);
+
+ // check num docs are the same
+ assertEquals(origShardToDocCount, AbstractCloudBackupRestoreTestCase.getShardToDocCountMap(solrClient, getCollectionState(restoreCollectionName)));
+
+ // this methods may get invoked multiple times, collection must be cleanup
+ CollectionAdminRequest.deleteCollection(restoreCollectionName).process(solrClient);
+ }
+
+
+ private void indexDocs(String collectionName, int numDocs, boolean useUUID) throws Exception {
+ Random random = new Random(docsSeed);
+
+ List<SolrInputDocument> docs = new ArrayList<>(numDocs);
+ for (int i=0; i<numDocs; i++) {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", (useUUID ? java.util.UUID.randomUUID().toString() : i));
+ doc.addField("shard_s", "shard" + (1 + random.nextInt(NUM_SHARDS))); // for implicit router
+ docs.add(doc);
+ }
+
+ CloudSolrClient client = cluster.getSolrClient();
+ client.add(collectionName, docs); //batch
+ client.commit(collectionName);
+
+ log.info("Indexed {} docs to collection: {}", numDocs, collectionName);
+ }
+
+ private int indexDocs(String collectionName, boolean useUUID) throws Exception {
+ Random random = new Random(docsSeed);// use a constant seed for the whole test run so that we can easily re-index.
+ int numDocs = random.nextInt(100) + 5;
+ indexDocs(collectionName, numDocs, useUUID);
+ return numDocs;
+ }
+
+ private class IncrementalBackupVerifier {
+ private BackupRepository repository;
+ private URI backupURI;
+ private String backupLocation;
+ private String backupName;
+ private BackupFilePaths incBackupFiles;
+
+ private Map<String, Collection<String>> lastShardCommitToBackupFiles = new HashMap<>();
+ // the first generation after calling backup is zero
+ private int numBackup = -1;
+ private int maxNumberOfBackupToKeep = 4;
+
+ IncrementalBackupVerifier(BackupRepository repository, String backupLocation,
+ String backupName, String collection, int maxNumberOfBackupToKeep) {
+ this.repository = repository;
+ this.backupLocation = backupLocation;
+ this.backupURI = repository.resolve(repository.createURI(backupLocation), backupName, collection);
+ this.incBackupFiles = new BackupFilePaths(repository, this.backupURI);
+ this.backupName = backupName;
+ this.maxNumberOfBackupToKeep = maxNumberOfBackupToKeep;
+ }
+
+ @SuppressWarnings("rawtypes")
+ private void backupThenWait() throws SolrServerException, IOException {
+ CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(getCollectionName(), backupName)
+ .setLocation(backupLocation)
+ .setIncremental(true)
+ .setMaxNumberBackupPoints(maxNumberOfBackupToKeep)
+ .setRepositoryName(BACKUP_REPO_NAME);
+ if (random().nextBoolean()) {
+ try {
+ RequestStatusState state = backup.processAndWait(cluster.getSolrClient(), 1000);
+ assertEquals(RequestStatusState.COMPLETED, state);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ numBackup++;
+ } else {
+ CollectionAdminResponse rsp = backup.process(cluster.getSolrClient());
+ assertEquals(0, rsp.getStatus());
+ NamedList resp = (NamedList) rsp.getResponse().get("response");
+ numBackup++;
+ assertEquals(numBackup, resp.get("backupId"));;
+ }
+ }
+
+ void incrementalBackupThenVerify() throws IOException, SolrServerException {
+ int numCopiedFiles = copiedFiles().size();
+ backupThenWait();
+ List<URI> newFilesCopiedOver = copiedFiles().subList(numCopiedFiles, copiedFiles().size());
+ verify(newFilesCopiedOver);
+ }
+
+ ShardBackupMetadata getLastShardBackupId(String shardName) throws IOException {
+ ShardBackupId shardBackupId = BackupProperties
+ .readFromLatest(repository, backupURI)
+ .flatMap(bp -> bp.getShardBackupIdFor(shardName))
+ .get();
+ return ShardBackupMetadata.from(repository, new BackupFilePaths(repository, backupURI).getShardBackupMetadataDir(), shardBackupId);
+ }
+
+ private void assertIndexInputEquals(IndexInput in1, IndexInput in2) throws IOException {
+ assertEquals(in1.length(), in2.length());
+ for (int i = 0; i < in1.length(); i++) {
+ assertEquals(in1.readByte(), in2.readByte());
+ }
+ }
+
+ private void assertFolderAreSame(URI uri1, URI uri2) throws IOException {
+ String[] files1 = repository.listAll(uri1);
+ String[] files2 = repository.listAll(uri2);
+ Arrays.sort(files1);
+ Arrays.sort(files2);
+
+ try {
+ assertArrayEquals(files1, files2);
+ } catch (AssertionError e) {
+ e.printStackTrace();
+ }
+
+ for (int i = 0; i < files1.length; i++) {
+ URI file1Uri = repository.resolve(uri1, files1[i]);
+ URI file2Uri = repository.resolve(uri2, files2[i]);
+ assertEquals(repository.getPathType(file1Uri), repository.getPathType(file2Uri));
+ if (repository.getPathType(file1Uri) == BackupRepository.PathType.DIRECTORY) {
+ assertFolderAreSame(file1Uri, file2Uri);
+ } else {
+ try (IndexInput in1 = repository.openInput(uri1, files1[i], IOContext.READONCE);
+ IndexInput in2 = repository.openInput(uri1, files1[i], IOContext.READONCE)) {
+ assertIndexInputEquals(in1, in2);
+ }
+ }
+ }
+ }
+
+ public void verify(List<URI> newFilesCopiedOver) throws IOException {
+ //Verify zk files are reuploaded to a appropriate each time a backup is called
+ //TODO make a little change to zk files and make sure that backed up files match with zk data
+ BackupId prevBackupId = new BackupId(Math.max(0, numBackup - 1));
+
+ URI backupPropertiesFile = repository.resolve(backupURI, "backup_"+numBackup+".properties");
+ URI zkBackupFolder = repository.resolve(backupURI, "zk_backup_"+numBackup);
+ assertTrue(repository.exists(backupPropertiesFile));
+ assertTrue(repository.exists(zkBackupFolder));
+ assertFolderAreSame(repository.resolve(backupURI, BackupFilePaths.getZkStateDir(prevBackupId)), zkBackupFolder);
+
+ // verify indexes file
+ for(Slice slice : getCollectionState(getCollectionName()).getSlices()) {
+ Replica leader = slice.getLeader();
+ final ShardBackupMetadata shardBackupMetadata = getLastShardBackupId(slice.getName());
+
+ try (SolrCore solrCore = cluster.getReplicaJetty(leader).getCoreContainer().getCore(leader.getCoreName())) {
+ Directory dir = solrCore.getDirectoryFactory().get(solrCore.getIndexDir(), DirectoryFactory.DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType);
+ try {
+ URI indexDir = incBackupFiles.getIndexDir();
+ IndexCommit lastCommit = solrCore.getDeletionPolicy().getLatestCommit();
+
+ Collection<String> newBackupFiles = newIndexFilesComparedToLastBackup(slice.getName(), lastCommit).stream()
+ .map(indexFile -> {
+ Optional<ShardBackupMetadata.BackedFile> backedFile = shardBackupMetadata.getFile(indexFile);
+ assertTrue(backedFile.isPresent());
+ return backedFile.get().uniqueFileName;
+ })
+ .collect(Collectors.toList());
+
+ lastCommit.getFileNames().forEach(
+ f -> {
+ Optional<ShardBackupMetadata.BackedFile> backedFile = shardBackupMetadata.getFile(f);
+ assertTrue(backedFile.isPresent());
+ String uniqueFileName = backedFile.get().uniqueFileName;
+
+ if (newBackupFiles.contains(uniqueFileName)) {
+ assertTrue(newFilesCopiedOver.contains(repository.resolve(indexDir, uniqueFileName)));
+ }
+
+ try {
+ Checksum localChecksum = repository.checksum(dir, f);
+ Checksum remoteChecksum = backedFile.get().fileChecksum;
+ assertEquals(localChecksum.checksum, remoteChecksum.checksum);
+ assertEquals(localChecksum.size, remoteChecksum.size);
+ } catch (IOException e) {
+ throw new AssertionError(e);
+ }
+ }
+ );
+
+ assertEquals("Incremental backup stored more files than needed", lastCommit.getFileNames().size(), shardBackupMetadata.listOriginalFileNames().size());
+ } finally {
+ solrCore.getDirectoryFactory().release(dir);
+ }
+ }
+ }
+ }
+
+ private Collection<String> newIndexFilesComparedToLastBackup(String shardName, IndexCommit currentCommit) throws IOException {
+ Collection<String> oldFiles = lastShardCommitToBackupFiles.put(shardName, currentCommit.getFileNames());
+ if (oldFiles == null)
+ oldFiles = new ArrayList<>();
+
+ List<String> newFiles = new ArrayList<>(currentCommit.getFileNames());
+ newFiles.removeAll(oldFiles);
+ return newFiles;
+ }
+ }
+}
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/package-info.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/package-info.java
new file mode 100644
index 00000000000..e0047cef9af
--- /dev/null
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Test framework classes for collection APIs
+ */
+package org.apache.solr.cloud.api.collections;
\ No newline at end of file
diff --git a/solr/test-framework/src/java/org/apache/solr/core/TrackingBackupRepository.java b/solr/test-framework/src/java/org/apache/solr/core/TrackingBackupRepository.java
new file mode 100644
index 00000000000..425ae91ae6c
--- /dev/null
+++ b/solr/test-framework/src/java/org/apache/solr/core/TrackingBackupRepository.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.core;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.backup.Checksum;
+import org.apache.solr.core.backup.repository.BackupRepository;
+import org.apache.solr.core.backup.repository.BackupRepositoryFactory;
+
+public class TrackingBackupRepository implements BackupRepository {
+ private static final List<URI> COPIED_FILES = Collections.synchronizedList(new ArrayList<>());
+
+ private BackupRepository delegate;
+
+ @Override
+ public <T> T getConfigProperty(String name) {
+ return delegate.getConfigProperty(name);
+ }
+
+ @Override
+ public URI createURI(String path) {
+ return delegate.createURI(path);
+ }
+
+ @Override
+ public URI resolve(URI baseUri, String... pathComponents) {
+ return delegate.resolve(baseUri, pathComponents);
+ }
+
+ @Override
+ public boolean exists(URI path) throws IOException {
+ return delegate.exists(path);
+ }
+
+ @Override
+ public PathType getPathType(URI path) throws IOException {
+ return delegate.getPathType(path);
+ }
+
+ @Override
+ public String[] listAll(URI path) throws IOException {
+ return delegate.listAll(path);
+ }
+
+ @Override
+ public IndexInput openInput(URI dirPath, String fileName, IOContext ctx) throws IOException {
+ return delegate.openInput(dirPath, fileName, ctx);
+ }
+
+ @Override
+ public OutputStream createOutput(URI path) throws IOException {
+ return delegate.createOutput(path);
+ }
+
+ @Override
+ public void createDirectory(URI path) throws IOException {
+ delegate.createDirectory(path);
+ }
+
+ @Override
+ public void deleteDirectory(URI path) throws IOException {
+ delegate.deleteDirectory(path);
+ }
+
+ @Override
+ public void copyIndexFileFrom(Directory sourceDir, String sourceFileName, URI destDir, String destFileName) throws IOException {
+ COPIED_FILES.add(delegate.resolve(destDir, destFileName));
+ delegate.copyIndexFileFrom(sourceDir, sourceFileName, destDir, destFileName);
+ }
+
+ @Override
+ public void close() throws IOException {
+ delegate.close();
+ }
+
+
+ @Override
+ public void delete(URI path, Collection<String> files, boolean ignoreNoSuchFileException) throws IOException {
+ delegate.delete(path, files, ignoreNoSuchFileException);
+ }
+
+ @Override
+ public Checksum checksum(Directory dir, String fileName) throws IOException {
+ return delegate.checksum(dir, fileName);
+ }
+
+ @Override
+ public void init(@SuppressWarnings("rawtypes") NamedList args) {
+ BackupRepositoryFactory factory = (BackupRepositoryFactory) args.get("factory");
+ SolrResourceLoader loader = (SolrResourceLoader) args.get("loader");
+ String repoName = (String) args.get("delegateRepoName");
+
+ this.delegate = factory.newInstance(loader, repoName);
+ }
+
+ /**
+ * @return list of files were copied by using {@link #copyFileFrom(Directory, String, URI)}
+ */
+ public static List<URI> copiedFiles() {
+ return new ArrayList<>(COPIED_FILES);
+ }
+
+ /**
+ * Clear all tracking data
+ */
+ public static void clear() {
+ COPIED_FILES.clear();
+ }
+
+ @Override
+ public void copyIndexFileTo(URI sourceRepo, String sourceFileName, Directory dest, String destFileName) throws IOException {
+ delegate.copyIndexFileTo(sourceRepo, sourceFileName, dest, destFileName);
+ }
+}