Apache Solr open-source search software
SOLR-13608: Fix incremental-backup test failures
Jason Gerlowskiaea29464f120
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.javaindex 9de6b6e2c8e..afa136845b6 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java@@ -16,19 +16,6 @@*/package org.apache.solr.cloud.api.collections;-import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;-import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;-import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES;-import static org.apache.solr.common.params.CommonAdminParams.ASYNC;-import static org.apache.solr.common.params.CommonParams.NAME;--import java.io.IOException;-import java.lang.invoke.MethodHandles;-import java.net.URI;-import java.util.Collection;-import java.util.Collections;-import java.util.Optional;-import org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.ShardRequestTracker;import org.apache.solr.common.SolrException;import org.apache.solr.common.SolrException.ErrorCode;@@ -44,6 +31,7 @@ import org.apache.solr.common.params.CoreAdminParams;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.core.CoreContainer;+import org.apache.solr.core.backup.BackupFilePaths;import org.apache.solr.core.backup.BackupManager;import org.apache.solr.core.backup.BackupProperties;import org.apache.solr.core.backup.ShardBackupId;@@ -52,11 +40,23 @@ import org.apache.solr.core.snapshots.CollectionSnapshotMetaData;import org.apache.solr.core.snapshots.CollectionSnapshotMetaData.CoreSnapshotMetaData;import org.apache.solr.core.snapshots.CollectionSnapshotMetaData.SnapshotStatus;import org.apache.solr.core.snapshots.SolrSnapshotManager;-import org.apache.solr.core.backup.BackupFilePaths;import org.apache.solr.handler.component.ShardHandler;import org.slf4j.Logger;import org.slf4j.LoggerFactory;+import java.io.IOException;+import java.lang.invoke.MethodHandles;+import java.net.URI;+import java.util.Collection;+import java.util.Collections;+import java.util.Optional;++import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;+import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;+import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES;+import static org.apache.solr.common.params.CommonAdminParams.ASYNC;+import static org.apache.solr.common.params.CommonParams.NAME;+public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());@@ -90,30 +90,26 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {try (BackupRepository repository = cc.newBackupRepository(repo)) {// Backup location- if (log.isDebugEnabled()) {- log.debug("Backup location received by overseer is: {}", message.getStr(CoreAdminParams.BACKUP_LOCATION));- }URI location = repository.createURI(message.getStr(CoreAdminParams.BACKUP_LOCATION));- log.debug("Resolved backup URI in overseer is: {}", location);- final URI backupPath = createAndValidateBackupPath(repository, incremental, location, backupName, collectionName);+ final URI backupUri = createAndValidateBackupPath(repository, incremental, location, backupName, collectionName);BackupManager backupMgr = (incremental) ?- BackupManager.forIncrementalBackup(repository, ocmh.zkStateReader, backupPath) :- BackupManager.forBackup(repository, ocmh.zkStateReader, backupPath);+ BackupManager.forIncrementalBackup(repository, ocmh.zkStateReader, backupUri) :+ BackupManager.forBackup(repository, ocmh.zkStateReader, backupUri);String strategy = message.getStr(CollectionAdminParams.INDEX_BACKUP_STRATEGY, CollectionAdminParams.COPY_FILES_STRATEGY);switch (strategy) {case CollectionAdminParams.COPY_FILES_STRATEGY: {if (incremental) {try {- incrementalCopyIndexFiles(backupPath, collectionName, message, results, backupProperties, backupMgr);+ incrementalCopyIndexFiles(backupUri, collectionName, message, results, backupProperties, backupMgr);} catch (SolrException e) {log.error("Error happened during incremental backup for collection:{}", collectionName, e);- ocmh.cleanBackup(repository, backupPath, backupMgr.getBackupId());+ ocmh.cleanBackup(repository, backupUri, backupMgr.getBackupId());throw e;}} else {- copyIndexFiles(backupPath, collectionName, message, results);+ copyIndexFiles(backupUri, collectionName, message, results);}break;}@@ -143,14 +139,13 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {int maxNumBackup = message.getInt(CoreAdminParams.MAX_NUM_BACKUP_POINTS, -1);if (incremental && maxNumBackup != -1) {- ocmh.deleteBackup(repository, backupPath, maxNumBackup, results);+ ocmh.deleteBackup(repository, backupUri, maxNumBackup, results);}}}private URI createAndValidateBackupPath(BackupRepository repository, boolean incremental, URI location, String backupName, String collection) throws IOException{final URI backupNamePath = repository.resolve(location, backupName);- log.debug("Location {} and backup-name {} combine to make path {}", location, backupName, backupNamePath);if ( (!incremental) && repository.exists(backupNamePath)) {throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The backup directory already exists: " + backupNamePath);@@ -209,7 +204,7 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {return r.get();}- private void incrementalCopyIndexFiles(URI backupPath, String collectionName, ZkNodeProps request,+ private void incrementalCopyIndexFiles(URI backupUri, String collectionName, ZkNodeProps request,NamedList<Object> results, BackupProperties backupProperties,BackupManager backupManager) throws IOException {String backupName = request.getStr(NAME);@@ -218,7 +213,7 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler();log.info("Starting backup of collection={} with backupName={} at location={}", collectionName, backupName,- backupPath);+ backupUri);Optional<BackupProperties> previousProps = backupManager.tryReadBackupProperties();final ShardRequestTracker shardRequestTracker = ocmh.asyncRequestTracker(asyncId);@@ -232,7 +227,7 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {}String coreName = replica.getStr(CORE_NAME_PROP);- ModifiableSolrParams params = coreBackupParams(backupPath, repoName, slice, coreName, true /* incremental backup */);+ ModifiableSolrParams params = coreBackupParams(backupUri, repoName, slice, coreName, true /* incremental backup */);params.set(CoreAdminParams.BACKUP_INCREMENTAL, true);previousProps.flatMap(bp -> bp.getShardBackupIdFor(slice.getName())).ifPresent(prevBackupPoint -> params.set(CoreAdminParams.PREV_SHARD_BACKUP_ID, prevBackupPoint.getIdAsString()));diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.javaindex 7709751d640..0a48f1df2e7 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java@@ -23,14 +23,14 @@ import org.apache.solr.common.cloud.ZkNodeProps;import org.apache.solr.common.params.CoreAdminParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.core.CoreContainer;-import org.apache.solr.core.backup.BackupId;import org.apache.solr.core.backup.AggregateBackupStats;+import org.apache.solr.core.backup.BackupFilePaths;+import org.apache.solr.core.backup.BackupId;import org.apache.solr.core.backup.BackupManager;import org.apache.solr.core.backup.BackupProperties;import org.apache.solr.core.backup.ShardBackupId;import org.apache.solr.core.backup.ShardBackupMetadata;import org.apache.solr.core.backup.repository.BackupRepository;-import org.apache.solr.core.backup.BackupFilePaths;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -117,10 +117,10 @@ public class DeleteBackupCmd implements OverseerCollectionMessageHandler.Cmd {deleteBackupIds(backupPath, repository, new HashSet<>(backupIdDeletes), results);}- void deleteBackupIds(URI backupPath, BackupRepository repository,+ void deleteBackupIds(URI backupUri, BackupRepository repository,Set<BackupId> backupIdsDeletes,@SuppressWarnings({"rawtypes"}) NamedList results) throws IOException {- BackupFilePaths incBackupFiles = new BackupFilePaths(repository, backupPath);+ BackupFilePaths incBackupFiles = new BackupFilePaths(repository, backupUri);URI shardBackupMetadataDir = incBackupFiles.getShardBackupMetadataDir();Set<String> referencedIndexFiles = new HashSet<>();@@ -167,15 +167,15 @@ public class DeleteBackupCmd implements OverseerCollectionMessageHandler.Cmd {repository.delete(incBackupFiles.getIndexDir(), unusedFiles, true);try {for (BackupId backupId : backupIdsDeletes) {- repository.deleteDirectory(repository.resolve(backupPath, BackupFilePaths.getZkStateDir(backupId)));+ repository.deleteDirectory(repository.resolve(backupUri, BackupFilePaths.getZkStateDir(backupId)));}} catch (FileNotFoundException e) {//ignore this}//add details to result before deleting backupPropFiles- addResult(backupPath, repository, backupIdsDeletes, backupIdToCollectionBackupPoint, results);- repository.delete(backupPath, backupIdsDeletes.stream().map(id -> BackupFilePaths.getBackupPropsName(id)).collect(Collectors.toList()), true);+ addResult(backupUri, repository, backupIdsDeletes, backupIdToCollectionBackupPoint, results);+ repository.delete(backupUri, backupIdsDeletes.stream().map(id -> BackupFilePaths.getBackupPropsName(id)).collect(Collectors.toList()), true);}@SuppressWarnings("unchecked")diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.javaindex b7f9c6d076a..53a02fa0886 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java@@ -16,24 +16,6 @@*/package org.apache.solr.cloud.api.collections;-import java.io.IOException;-import java.lang.invoke.MethodHandles;-import java.net.URI;-import java.util.ArrayList;-import java.util.Arrays;-import java.util.Collection;-import java.util.Collections;-import java.util.HashMap;-import java.util.List;-import java.util.Map;-import java.util.Random;-import java.util.Set;-import java.util.concurrent.ConcurrentHashMap;-import java.util.concurrent.ExecutorService;-import java.util.concurrent.SynchronousQueue;-import java.util.concurrent.TimeUnit;-import java.util.concurrent.TimeoutException;-import com.google.common.collect.ImmutableMap;import org.apache.commons.lang3.StringUtils;import org.apache.solr.client.solrj.SolrResponse;@@ -95,16 +77,25 @@ import org.apache.zookeeper.KeeperException;import org.slf4j.Logger;import org.slf4j.LoggerFactory;-import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;-import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;-import static org.apache.solr.common.cloud.ZkStateReader.CORE_NODE_NAME_PROP;-import static org.apache.solr.common.cloud.ZkStateReader.ELECTION_NODE_PROP;-import static org.apache.solr.common.cloud.ZkStateReader.NODE_NAME_PROP;-import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_PROP;-import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_VALUE_PROP;-import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP;-import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;-import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;+import java.io.IOException;+import java.lang.invoke.MethodHandles;+import java.net.URI;+import java.util.ArrayList;+import java.util.Arrays;+import java.util.Collection;+import java.util.Collections;+import java.util.HashMap;+import java.util.List;+import java.util.Map;+import java.util.Random;+import java.util.Set;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.SynchronousQueue;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;++import static org.apache.solr.common.cloud.ZkStateReader.*;import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION;import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;import static org.apache.solr.common.params.CommonAdminParams.ASYNC;@@ -630,9 +621,9 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,}@SuppressWarnings({"rawtypes"})- void cleanBackup(BackupRepository repository, URI backupPath, BackupId backupId) throws Exception {+ void cleanBackup(BackupRepository repository, URI backupUri, BackupId backupId) throws Exception {((DeleteBackupCmd)commandMap.get(DELETEBACKUP))- .deleteBackupIds(backupPath, repository, Collections.singleton(backupId), new NamedList());+ .deleteBackupIds(backupUri, repository, Collections.singleton(backupId), new NamedList());}void deleteBackup(BackupRepository repository, URI backupPath,diff --git a/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java b/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.javaindex 1fcdec72ad8..e8cfef5c275 100644--- a/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java+++ b/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java@@ -17,9 +17,18 @@package org.apache.solr.core.backup.repository;+import com.google.common.base.Preconditions;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.FSDirectory;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.NIOFSDirectory;+import org.apache.lucene.store.NoLockFactory;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.core.DirectoryFactory;+import java.io.IOException;import java.io.OutputStream;-import java.lang.invoke.MethodHandles;import java.net.URI;import java.net.URISyntaxException;import java.nio.file.FileVisitResult;@@ -33,26 +42,12 @@ import java.nio.file.attribute.BasicFileAttributes;import java.util.Collection;import java.util.Objects;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.FSDirectory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.store.NIOFSDirectory;-import org.apache.lucene.store.NoLockFactory;-import org.apache.solr.common.util.NamedList;-import org.apache.solr.core.DirectoryFactory;--import com.google.common.base.Preconditions;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;-/*** A concrete implementation of {@linkplain BackupRepository} interface supporting backup/restore of Solr indexes to a* local file-system. (Note - This can even be used for a shared file-system if it is exposed via a local file-system* interface e.g. NFS).*/public class LocalFileSystemRepository implements BackupRepository {- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());@SuppressWarnings("rawtypes")private NamedList config = null;@@ -71,7 +66,6 @@ public class LocalFileSystemRepository implements BackupRepository {@Overridepublic URI createURI(String location) {Objects.requireNonNull(location);- log.debug("Creating URI from location: {}", location);URI result = null;try {@@ -83,14 +77,12 @@ public class LocalFileSystemRepository implements BackupRepository {result = Paths.get(location).toUri();}- log.debug("Created URI is: {}", result);return result;}@Overridepublic URI resolve(URI baseUri, String... pathComponents) {Preconditions.checkArgument(pathComponents.length > 0);- log.debug("Resolving URI from base: {}", baseUri);Path result = Paths.get(baseUri);for (int i = 0; i < pathComponents.length; i++) {@@ -103,9 +95,6 @@ public class LocalFileSystemRepository implements BackupRepository {}- if (log.isDebugEnabled()) {- log.debug("Resolved URI is: {}", result.toUri());- }return result.toUri();}@@ -136,7 +125,6 @@ public class LocalFileSystemRepository implements BackupRepository {@Overridepublic boolean exists(URI path) throws IOException {- log.debug("Checking whether URI exists: {}", path);return Files.exists(Paths.get(path));}diff --git a/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java b/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.javaindex c2c78066a74..5543f048b5f 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java@@ -60,7 +60,7 @@ class BackupCoreOp implements CoreAdminHandler.CoreAdminOp {if (incremental) {if ("file".equals(locationUri.getScheme())) {- core.getCoreContainer().assertPathAllowed(Paths.get(location));+ core.getCoreContainer().assertPathAllowed(Paths.get(locationUri));}final ShardBackupId prevShardBackupId = prevShardBackupIdStr != null ? ShardBackupId.from(prevShardBackupIdStr) : null;BackupFilePaths incBackupFiles = new BackupFilePaths(repository, locationUri);diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.javaindex 60f56d11be6..54aaf281e40 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java@@ -1028,12 +1028,10 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission+ " parameter or as a default repository property or as a cluster property.");}}- log.debug("Provided backup location is: {}", location);boolean incremental = req.getParams().getBool(CoreAdminParams.BACKUP_INCREMENTAL, true);// Check if the specified location is valid for this repository.final URI uri = repository.createURI(location);- log.debug("Provided backup location uri is: {}", uri);try {if (!repository.exists(uri)) {throw new SolrException(ErrorCode.SERVER_ERROR, "specified location " + uri + " does not exist.");diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.javaindex b247ae68925..b552c6c5689 100644--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java@@ -20,8 +20,9 @@ package org.apache.solr.cloud.api.collections;import org.junit.BeforeClass;public class LocalFSCloudIncrementalBackupTest extends AbstractIncrementalBackupTest {- public static final String SOLR_XML = "<solr>\n" ++ private static final String SOLR_XML = "<solr>\n" +"\n" ++ " <str name=\"allowPaths\">ALLOWPATHS_TEMPLATE_VAL</str>\n" +" <str name=\"shareSchema\">${shareSchema:false}</str>\n" +" <str name=\"configSetBaseDir\">${configSetBaseDir:configsets}</str>\n" +" <str name=\"coreRootDirectory\">${coreRootDirectory:.}</str>\n" +@@ -57,17 +58,17 @@ public class LocalFSCloudIncrementalBackupTest extends AbstractIncrementalBackup@BeforeClasspublic static void setupClass() throws Exception {- configureCluster(NUM_SHARDS)// nodes- .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))- .withSolrXml(SOLR_XML)- .configure();-boolean whitespacesInPath = random().nextBoolean();if (whitespacesInPath) {backupLocation = createTempDir("my backup").toAbsolutePath().toString();} else {backupLocation = createTempDir("mybackup").toAbsolutePath().toString();}++ configureCluster(NUM_SHARDS)// nodes+ .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))+ .withSolrXml(SOLR_XML.replace("ALLOWPATHS_TEMPLATE_VAL", backupLocation))+ .configure();}@Overridediff --git a/solr/core/src/test/org/apache/solr/handler/TestIncrementalCoreBackup.java b/solr/core/src/test/org/apache/solr/handler/TestIncrementalCoreBackup.javaindex d8301da3846..5035827332d 100644--- a/solr/core/src/test/org/apache/solr/handler/TestIncrementalCoreBackup.java+++ b/solr/core/src/test/org/apache/solr/handler/TestIncrementalCoreBackup.java@@ -27,18 +27,16 @@ import org.apache.solr.core.backup.ShardBackupMetadata;import org.apache.solr.core.backup.repository.BackupRepository;import org.apache.solr.handler.admin.CoreAdminHandler;import org.apache.solr.response.SolrQueryResponse;-import org.apache.solr.util.LogLevel;import org.junit.After;import org.junit.Before;import org.junit.Test;-import java.io.File;import java.io.IOException;import java.net.URI;+import java.nio.file.Path;import java.nio.file.Paths;import java.util.Arrays;-@LogLevel("org.apache.solr.cloud.api.collections.BackupCmd=DEBUG;org.apache.solr.handler.admin.BackupCoreOp=DEBUG;org.apache.solr.core.backup.repository.LocalFileSystemRepository=DEBUG")public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {@Before // unique core per testpublic void coreInit() throws Exception {@@ -64,21 +62,22 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {assertQ(req("q", "id:2"), "//result[@numFound='0']");//call backup- final URI location = createAndBootstrapLocationForBackup();+ final Path locationPath = createBackupLocation();+ final URI locationUri = bootstrapBackupLocation(locationPath);final ShardBackupId shardBackupId = new ShardBackupId("shard1", BackupId.zero());final CoreContainer cores = h.getCoreContainer();- cores.getAllowPaths().add(Paths.get(location));+ cores.getAllowPaths().add(Paths.get(locationUri));try (final CoreAdminHandler admin = new CoreAdminHandler(cores)) {SolrQueryResponse resp = new SolrQueryResponse();admin.handleRequestBody(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),"core", DEFAULT_TEST_COLLECTION_NAME,- "location", location.getPath(),+ "location", locationPath.toString(),CoreAdminParams.SHARD_BACKUP_ID, shardBackupId.getIdAsString()), resp);assertNull("Backup should have succeeded", resp.getException());- simpleBackupCheck(location, shardBackupId);+ simpleBackupCheck(locationUri, shardBackupId);}}@@ -102,7 +101,8 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {final CoreContainer cores = h.getCoreContainer();final CoreAdminHandler admin = new CoreAdminHandler(cores);- final URI location = createAndBootstrapLocationForBackup();+ final Path locationPath = createBackupLocation();+ final URI locationUri = bootstrapBackupLocation(locationPath);final ShardBackupId firstShardBackup = new ShardBackupId("shard1", BackupId.zero());{ // first a backup before we've ever done *anything*...@@ -110,11 +110,11 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {admin.handleRequestBody(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),"core", DEFAULT_TEST_COLLECTION_NAME,- "location", location.getPath(),+ "location", locationPath.toString(),CoreAdminParams.SHARD_BACKUP_ID, firstShardBackup.getIdAsString()),resp);assertNull("Backup should have succeeded", resp.getException());- simpleBackupCheck(location, firstShardBackup, initialEmptyIndexSegmentFileName);+ simpleBackupCheck(locationUri, firstShardBackup, initialEmptyIndexSegmentFileName);}{ // Empty (named) snapshot..@@ -135,11 +135,11 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {admin.handleRequestBody(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),"core", DEFAULT_TEST_COLLECTION_NAME,- "location", location.getPath(),+ "location", locationPath.toString(),CoreAdminParams.SHARD_BACKUP_ID, secondShardBackupId.getIdAsString()),resp);assertNull("Backup should have succeeded", resp.getException());- simpleBackupCheck(location, secondShardBackupId, initialEmptyIndexSegmentFileName);+ simpleBackupCheck(locationUri, secondShardBackupId, initialEmptyIndexSegmentFileName);}{ // Second empty (named) snapshot..@@ -156,7 +156,7 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {assertU(commit());for (ShardBackupId shardBackupId: Arrays.asList(firstShardBackup, secondShardBackupId)) {- simpleBackupCheck(location, shardBackupId, initialEmptyIndexSegmentFileName);+ simpleBackupCheck(locationUri, shardBackupId, initialEmptyIndexSegmentFileName);}// Make backups from each of the snapshots and check they are still empty as well...@@ -167,11 +167,11 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),"core", DEFAULT_TEST_COLLECTION_NAME,"commitName", "empty_snapshotA",- "location", location.getPath(),+ "location", locationPath.toString(),CoreAdminParams.SHARD_BACKUP_ID, thirdShardBackup.getIdAsString()),resp);assertNull("Backup from snapshot empty_snapshotA should have succeeded", resp.getException());- simpleBackupCheck(location, thirdShardBackup, initialEmptyIndexSegmentFileName);+ simpleBackupCheck(locationUri, thirdShardBackup, initialEmptyIndexSegmentFileName);}{final ShardBackupId fourthShardBackup = new ShardBackupId("shard1", new BackupId(3));@@ -180,11 +180,11 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),"core", DEFAULT_TEST_COLLECTION_NAME,"commitName", "empty_snapshotB",- "location", location.getPath(),+ "location", locationPath.toString(),CoreAdminParams.SHARD_BACKUP_ID, fourthShardBackup.getIdAsString()),resp);assertNull("Backup from snapshot empty_snapshotB should have succeeded", resp.getException());- simpleBackupCheck(location, fourthShardBackup, initialEmptyIndexSegmentFileName);+ simpleBackupCheck(locationUri, fourthShardBackup, initialEmptyIndexSegmentFileName);}admin.close();}@@ -211,7 +211,8 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {final CoreContainer cores = h.getCoreContainer();final CoreAdminHandler admin = new CoreAdminHandler(cores);- final URI location = createAndBootstrapLocationForBackup();+ final Path locationPath = createBackupLocation();+ final URI locationUri = bootstrapBackupLocation(locationPath);final ShardBackupId firstShardBackupId = new ShardBackupId("shard1", BackupId.zero());{ // take an initial 'backup1a' containing our 1 document@@ -220,11 +221,11 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),"core", DEFAULT_TEST_COLLECTION_NAME,"name", "backup1a",- "location", location.getPath(),+ "location", locationPath.toString(),CoreAdminParams.SHARD_BACKUP_ID, firstShardBackupId.getIdAsString()),resp);assertNull("Backup should have succeeded", resp.getException());- simpleBackupCheck(location, firstShardBackupId, oneDocSegmentFile);+ simpleBackupCheck(locationUri, firstShardBackupId, oneDocSegmentFile);}{ // and an initial "snapshot1a' that should eventually match@@ -253,11 +254,11 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {admin.handleRequestBody(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),"core", DEFAULT_TEST_COLLECTION_NAME,- "location", location.getPath(),+ "location", locationPath.toString(),CoreAdminParams.SHARD_BACKUP_ID, secondShardBackupId.getIdAsString()),resp);assertNull("Backup should have succeeded", resp.getException());- simpleBackupCheck(location, secondShardBackupId, oneDocSegmentFile);+ simpleBackupCheck(locationUri, secondShardBackupId, oneDocSegmentFile);}{ // and a second "snapshot1b' should also still be identical@@ -272,8 +273,8 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {// Hard Committing the 2nd doc now should not affect the existing backups or snapshots...assertU(commit());- simpleBackupCheck(location, firstShardBackupId, oneDocSegmentFile); // backup1a- simpleBackupCheck(location, secondShardBackupId, oneDocSegmentFile); // backup1b+ simpleBackupCheck(locationUri, firstShardBackupId, oneDocSegmentFile); // backup1a+ simpleBackupCheck(locationUri, secondShardBackupId, oneDocSegmentFile); // backup1bfinal ShardBackupId thirdShardBackupId = new ShardBackupId("shard1", new BackupId(2));{ // But we should be able to confirm both docs appear in a new backup (not based on a previous snapshot)@@ -281,13 +282,13 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {admin.handleRequestBody(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),"core", DEFAULT_TEST_COLLECTION_NAME,- "location", location.getPath(),+ "location", locationPath.toString(),CoreAdminParams.SHARD_BACKUP_ID, thirdShardBackupId.getIdAsString()),resp);assertNull("Backup should have succeeded", resp.getException());// TODO This doesn't actually check that backup has both docs! Can we do better than this without doing a full restore?// Maybe validate the new segments_X file at least to show that it's picked up the latest commit?- simpleBackupCheck(location, thirdShardBackupId);+ simpleBackupCheck(locationUri, thirdShardBackupId);}// if we go back and create backups from our earlier snapshots they should still only@@ -300,11 +301,11 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),"core", DEFAULT_TEST_COLLECTION_NAME,"commitName", "snapshot1a",- "location", location.getPath(),+ "location", locationPath.toString(),CoreAdminParams.SHARD_BACKUP_ID, fourthShardBackupId.getIdAsString()),resp);assertNull("Backup of snapshot1a should have succeeded", resp.getException());- simpleBackupCheck(location, fourthShardBackupId, oneDocSegmentFile);+ simpleBackupCheck(locationUri, fourthShardBackupId, oneDocSegmentFile);}final ShardBackupId fifthShardBackupId = new ShardBackupId("shard1", new BackupId(4));{@@ -313,11 +314,11 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),"core", DEFAULT_TEST_COLLECTION_NAME,"commitName", "snapshot1b",- "location", location.getPath(),+ "location", locationPath.toString(),CoreAdminParams.SHARD_BACKUP_ID, fifthShardBackupId.getIdAsString()),resp);assertNull("Backup of snapshot1b should have succeeded", resp.getException());- simpleBackupCheck(location, fifthShardBackupId, oneDocSegmentFile);+ simpleBackupCheck(locationUri, fifthShardBackupId, oneDocSegmentFile);}admin.close();@@ -355,15 +356,18 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {}}- private URI createAndBootstrapLocationForBackup() throws IOException {- final File locationFile = createTempDir().toFile();- final String location = locationFile.getAbsolutePath();+ private Path createBackupLocation() {+ return createTempDir().toAbsolutePath();+ }- h.getCoreContainer().getAllowPaths().add(locationFile.toPath());+ private URI bootstrapBackupLocation(Path locationPath) throws IOException {+ final String locationPathStr = locationPath.toString();+ h.getCoreContainer().getAllowPaths().add(locationPath);try (BackupRepository backupRepo = h.getCoreContainer().newBackupRepository(null)) {- final BackupFilePaths backupFilePaths = new BackupFilePaths(backupRepo, backupRepo.createURI(location));+ final URI locationUri = backupRepo.createURI(locationPathStr);+ final BackupFilePaths backupFilePaths = new BackupFilePaths(backupRepo, locationUri);backupFilePaths.createIncrementalBackupFolders();- return backupRepo.createURI(location);+ return locationUri;}}}diff --git a/solr/core/src/test/org/apache/solr/handler/TestStressIncrementalBackup.java b/solr/core/src/test/org/apache/solr/handler/TestStressIncrementalBackup.javaindex 871d3498be6..0e507a427a7 100644--- a/solr/core/src/test/org/apache/solr/handler/TestStressIncrementalBackup.java+++ b/solr/core/src/test/org/apache/solr/handler/TestStressIncrementalBackup.java@@ -17,11 +17,6 @@package org.apache.solr.handler;-import java.io.File;-import java.lang.invoke.MethodHandles;-import java.util.concurrent.atomic.AtomicBoolean;-import java.util.concurrent.atomic.AtomicReference;-import org.apache.lucene.util.LuceneTestCase;import org.apache.solr.client.solrj.SolrClient;import org.apache.solr.client.solrj.request.CollectionAdminRequest;@@ -32,26 +27,31 @@ import org.apache.solr.client.solrj.response.UpdateResponse;import org.apache.solr.cloud.SolrCloudTestCase;import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.params.UpdateParams;-import org.apache.solr.util.LogLevel;import org.junit.After;import org.junit.Before;import org.slf4j.Logger;import org.slf4j.LoggerFactory;+import java.lang.invoke.MethodHandles;+import java.nio.file.Path;+import java.util.concurrent.atomic.AtomicBoolean;+import java.util.concurrent.atomic.AtomicReference;+import static org.apache.solr.handler.TestStressThreadBackup.makeDoc;//@LuceneTestCase.Nightly@LuceneTestCase.SuppressCodecs({"SimpleText"})-@LogLevel("org.apache.solr.handler.SnapShooter=DEBUG;org.apache.solr.core.IndexDeletionPolicyWrapper=DEBUG")public class TestStressIncrementalBackup extends SolrCloudTestCase {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());- private File backupDir;+ private Path backupPath;private SolrClient adminClient;private SolrClient coreClient;+@Beforepublic void beforeTest() throws Exception {- backupDir = createTempDir(getTestClass().getSimpleName() + "_backups").toFile();+ backupPath = createTempDir(getTestClass().getSimpleName() + "_backups");+ System.setProperty("solr.allowPaths", backupPath.toString());// NOTE: we don't actually care about using SolrCloud, but we want to use SolrClient and I can't// bring myself to deal with the nonsense that is SolrJettyTestBase.@@ -87,6 +87,8 @@ public class TestStressIncrementalBackup extends SolrCloudTestCase {if (null != coreClient) {coreClient.close();}++ System.clearProperty("solr.allowPaths");}public void testCoreAdminHandler() throws Exception {@@ -158,7 +160,7 @@ public class TestStressIncrementalBackup extends SolrCloudTestCase {public void makeBackup() throws Exception {CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(DEFAULT_TEST_COLLECTION_NAME, "stressBackup")- .setLocation(backupDir.getAbsolutePath())+ .setLocation(backupPath.toString()).setIncremental(true).setMaxNumberBackupPoints(5);if (random().nextBoolean()) {diff --git a/solr/core/src/test/org/apache/solr/handler/TestStressThreadBackup.java b/solr/core/src/test/org/apache/solr/handler/TestStressThreadBackup.javaindex 8c7128bdaf8..f8f33a3ee17 100644--- a/solr/core/src/test/org/apache/solr/handler/TestStressThreadBackup.java+++ b/solr/core/src/test/org/apache/solr/handler/TestStressThreadBackup.java@@ -37,7 +37,6 @@ import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.params.UpdateParams;import org.apache.solr.common.util.TimeSource;-import org.apache.solr.util.LogLevel;import org.apache.solr.util.TimeOut;import org.junit.After;import org.junit.AfterClass;@@ -63,7 +62,6 @@ import java.util.regex.Pattern;@Nightly@SuppressCodecs({"SimpleText"})-@LogLevel("org.apache.solr.handler.SnapShooter=DEBUG;org.apache.solr.core.IndexDeletionPolicyWrapper=DEBUG")public class TestStressThreadBackup extends SolrCloudTestCase {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());@@ -369,7 +367,8 @@ public class TestStressThreadBackup extends SolrCloudTestCase {public void makeBackup(final String backupName, final String snapName) throws Exception {ModifiableSolrParams p = params(CoreAdminParams.CORE, coreName,CoreAdminParams.NAME, backupName,- CoreAdminParams.BACKUP_LOCATION, backupDir.getAbsolutePath());+ CoreAdminParams.BACKUP_LOCATION, backupDir.getAbsolutePath(),+ CoreAdminParams.BACKUP_INCREMENTAL, "false");if (null != snapName) {p.add(CoreAdminParams.COMMIT_NAME, snapName);}