Apache Solr open-source search software
SOLR-16852: Let backups have custom key/values (#1739)
Tomas Eduardo Fernandez Lobbef88891e6bb80
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex bc1564fc028..ef7e37ea61e 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -105,6 +105,8 @@ New Features* SOLR-15367: Convert "rid" functionality into a default Tracer (Alex Deparvu, David Smiley)+* SOLR-16852: Backups now allow metadata to be added as key-values (Tomás Fernández Löbbe)+Improvements---------------------* SOLR-16490: `/admin/cores?action=backupcore` now has a v2 equivalent, available atdiff --git a/solr/api/src/java/org/apache/solr/client/api/model/CollectionBackupDetails.java b/solr/api/src/java/org/apache/solr/client/api/model/CollectionBackupDetails.javaindex a46529baab4..94d18f8682f 100644--- a/solr/api/src/java/org/apache/solr/client/api/model/CollectionBackupDetails.java+++ b/solr/api/src/java/org/apache/solr/client/api/model/CollectionBackupDetails.java@@ -35,4 +35,5 @@ public class CollectionBackupDetails {public String configsetName;@JsonProperty public String collectionAlias;+ @JsonProperty public Map<String, String> extraProperties;}diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.javaindex 5d4aaf7fd79..151cbeeee1a 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java@@ -29,6 +29,7 @@ import java.util.ArrayList;import java.util.Collection;import java.util.Collections;import java.util.List;+import java.util.Map;import java.util.Optional;import org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ShardRequestTracker;import org.apache.solr.common.SolrException;@@ -68,6 +69,7 @@ public class BackupCmd implements CollApiCmds.CollectionApiCommand {this.ccc = ccc;}+ @SuppressWarnings("unchecked")@Overridepublic void call(ClusterState state, ZkNodeProps message, NamedList<Object> results)throws Exception {@@ -91,8 +93,11 @@ public class BackupCmd implements CollApiCmds.CollectionApiCommand {.getCollection(collectionName).getConfigName();+ Map<String, String> customProps = (Map<String, String>) message.get("extraProperties");+BackupProperties backupProperties =- BackupProperties.create(backupName, collectionName, extCollectionName, configName);+ BackupProperties.create(+ backupName, collectionName, extCollectionName, configName, customProps);CoreContainer cc = ccc.getCoreContainer();try (BackupRepository repository = cc.newBackupRepository(repo)) {@@ -367,6 +372,9 @@ public class BackupCmd implements CollApiCmds.CollectionApiCommand {}aggRsp.add("indexVersion", backupProps.getIndexVersion());aggRsp.add("startTime", backupProps.getStartTime());+ if (backupProps.getExtraProperties() != null) {+ aggRsp.add("extraProperties", backupProps.getExtraProperties());+ }// Optional options for backupsOptional<Integer> indexFileCount = Optional.empty();diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupProperties.java b/solr/core/src/java/org/apache/solr/core/backup/BackupProperties.javaindex 1e452bb516d..dcc8509b574 100644--- a/solr/core/src/java/org/apache/solr/core/backup/BackupProperties.java+++ b/solr/core/src/java/org/apache/solr/core/backup/BackupProperties.java@@ -48,18 +48,33 @@ import org.apache.solr.util.PropertiesInputStream;*/public class BackupProperties {+ private static final String EXTRA_PROPERTY_PREFIX = "property.";+private double indexSizeMB;private int indexFileCount;- private Properties properties;+ private final Properties properties;++ private final Map<String, String> extraProperties;- private BackupProperties(Properties properties) {+ private BackupProperties(Properties properties, Map<String, String> extraProperties) {this.properties = properties;+ if (extraProperties == null) {+ extraProperties = Map.of();+ } else if (extraProperties.keySet().stream().anyMatch(String::isEmpty)) {+ throw new IllegalArgumentException("Can't have an extra property with an empty key");+ }+ this.extraProperties = extraProperties;}public static BackupProperties create(- String backupName, String collectionName, String extCollectionName, String configName) {- Properties properties = new Properties();+ String backupName,+ String collectionName,+ String extCollectionName,+ String configName,+ Map<String, String> extraProperties) {+ final Properties properties = new Properties();+properties.put(BackupManager.BACKUP_NAME_PROP, backupName);properties.put(BackupManager.COLLECTION_NAME_PROP, collectionName);properties.put(BackupManager.COLLECTION_ALIAS_PROP, extCollectionName);@@ -67,7 +82,7 @@ public class BackupProperties {properties.put(BackupManager.START_TIME_PROP, Instant.now().toString());properties.put(BackupManager.INDEX_VERSION_PROP, Version.LATEST.toString());- return new BackupProperties(properties);+ return new BackupProperties(properties, extraProperties);}public static Optional<BackupProperties> readFromLatest(@@ -93,10 +108,29 @@ public class BackupProperties {repository.openInput(backupPath, fileName, IOContext.DEFAULT)),StandardCharsets.UTF_8)) {props.load(is);- return new BackupProperties(props);+ Map<String, String> extraProperties = extractExtraProperties(props);+ return new BackupProperties(props, extraProperties);}}+ private static Map<String, String> extractExtraProperties(Properties props) {+ Map<String, String> extraProperties = new HashMap<>();+ props+ .entrySet()+ .removeIf(+ e -> {+ String entryKey = e.getKey().toString();+ if (entryKey.startsWith(EXTRA_PROPERTY_PREFIX)) {+ extraProperties.put(+ entryKey.substring(EXTRA_PROPERTY_PREFIX.length()),+ String.valueOf(e.getValue()));+ return true;+ }+ return false;+ });+ return extraProperties;+ }+public List<String> getAllShardBackupMetadataFiles() {return properties.entrySet().stream().filter(entry -> entry.getKey().toString().endsWith(".md"))@@ -128,10 +162,14 @@ public class BackupProperties {}public void store(Writer propsWriter) throws IOException {- properties.put("indexSizeMB", String.valueOf(indexSizeMB));- properties.put("indexFileCount", String.valueOf(indexFileCount));- properties.put(BackupManager.END_TIME_PROP, Instant.now().toString());- properties.store(propsWriter, "Backup properties file");+ Properties propertiesCopy = (Properties) properties.clone();+ propertiesCopy.put("indexSizeMB", String.valueOf(indexSizeMB));+ propertiesCopy.put("indexFileCount", String.valueOf(indexFileCount));+ propertiesCopy.put(BackupManager.END_TIME_PROP, Instant.now().toString());+ if (extraProperties != null && !extraProperties.isEmpty()) {+ extraProperties.forEach((k, v) -> propertiesCopy.put(EXTRA_PROPERTY_PREFIX + k, v));+ }+ propertiesCopy.store(propsWriter, "Backup properties file");}public String getCollection() {@@ -158,14 +196,20 @@ public class BackupProperties {return properties.getProperty(BackupManager.INDEX_VERSION_PROP);}+ public Map<String, String> getExtraProperties() {+ return extraProperties;+ }+public Map<String, Object> getDetails() {final Map<String, Object> result = new HashMap<>();- properties.entrySet().stream()- .forEach(entry -> result.put(entry.getKey().toString(), entry.getValue()));+ properties.entrySet().forEach(entry -> result.put(entry.getKey().toString(), entry.getValue()));result.remove(BackupManager.BACKUP_NAME_PROP);result.remove(BackupManager.COLLECTION_NAME_PROP);result.put("indexSizeMB", Double.valueOf(properties.getProperty("indexSizeMB")));result.put("indexFileCount", Integer.valueOf(properties.getProperty("indexFileCount")));+ if (extraProperties != null && !extraProperties.isEmpty()) {+ result.put("extraProperties", extraProperties);+ }Map<String, String> shardBackupIds = new HashMap<>();Iterator<String> keyIt = result.keySet().iterator();diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackupAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackupAPI.javaindex 0339b29a887..d37688f2162 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackupAPI.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackupAPI.java@@ -22,6 +22,7 @@ import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION;import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES;import static org.apache.solr.common.params.CollectionAdminParams.INDEX_BACKUP_STRATEGY;+import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_PREFIX;import static org.apache.solr.common.params.CommonAdminParams.ASYNC;import static org.apache.solr.common.params.CommonParams.NAME;import static org.apache.solr.common.params.CoreAdminParams.BACKUP_CONFIGSET;@@ -31,6 +32,7 @@ import static org.apache.solr.common.params.CoreAdminParams.BACKUP_REPOSITORY;import static org.apache.solr.common.params.CoreAdminParams.COMMIT_NAME;import static org.apache.solr.common.params.CoreAdminParams.MAX_NUM_BACKUP_POINTS;import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT;+import static org.apache.solr.handler.admin.api.CreateCollection.copyPrefixedPropertiesWithoutPrefix;import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM;import com.fasterxml.jackson.annotation.JsonProperty;@@ -166,6 +168,9 @@ public class CreateCollectionBackupAPI extends BackupAPIBase {requestBody.incremental = params.getBool(BACKUP_INCREMENTAL);requestBody.backupConfigset = params.getBool(BACKUP_CONFIGSET);requestBody.maxNumBackupPoints = params.getInt(MAX_NUM_BACKUP_POINTS);+ requestBody.extraProperties =+ copyPrefixedPropertiesWithoutPrefix(params, new HashMap<>(), PROPERTY_PREFIX);+requestBody.async = params.get(ASYNC);return requestBody;@@ -192,6 +197,7 @@ public class CreateCollectionBackupAPI extends BackupAPIBase {@JsonProperty public Boolean backupConfigset;@JsonProperty public Integer maxNumBackupPoints;@JsonProperty public String async;+ @JsonProperty public Map<String, String> extraProperties;}public static class CreateCollectionBackupResponseBody@@ -215,6 +221,7 @@ public class CreateCollectionBackupAPI extends BackupAPIBase {@JsonProperty public Integer indexFileCount;@JsonProperty public Integer uploadedIndexFileCount;@JsonProperty public Double indexSizeMB;+ @JsonProperty public Map<String, String> extraProperties;@JsonProperty("uploadedIndexFileMB")public Double uploadedIndexSizeMB;diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.javaindex c6e52ffeab6..869c90fd67d 100644--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java@@ -17,8 +17,19 @@package org.apache.solr.cloud.api.collections;+import java.util.List;+import java.util.Map;+import java.util.Properties;import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.solr.client.solrj.SolrQuery;+import org.apache.solr.client.solrj.impl.CloudSolrClient;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.response.CollectionAdminResponse;+import org.apache.solr.cloud.AbstractDistribZkTestBase;+import org.apache.solr.common.cloud.ZkStateReader;+import org.apache.solr.core.backup.repository.BackupRepository;import org.junit.BeforeClass;+import org.junit.Test;// Backups do checksum validation against a footer value not present in 'SimpleText'@LuceneTestCase.SuppressCodecs({"SimpleText"})@@ -84,4 +95,61 @@ public class LocalFSCloudIncrementalBackupTest extends AbstractIncrementalBackuppublic String getBackupLocation() {return backupLocation;}++ @SuppressWarnings("unchecked")+ @Test+ public void testCustomProperties() throws Exception {+ setTestSuffix("testCustomProperties");+ final String backupCollectionName = getCollectionName();+ final String restoreCollectionName = backupCollectionName + "_restore";++ CloudSolrClient solrClient = cluster.getSolrClient();++ CollectionAdminRequest.createCollection(backupCollectionName, "conf1", NUM_SHARDS, 1)+ .process(solrClient);+ int numDocs = indexDocs(backupCollectionName, true);+ String backupName = BACKUPNAME_PREFIX + testSuffix;+ try (BackupRepository repository =+ cluster.getJettySolrRunner(0).getCoreContainer().newBackupRepository(BACKUP_REPO_NAME)) {+ String backupLocation = repository.getBackupLocation(getBackupLocation());+ Properties extraProps = new Properties();+ extraProps.putAll(Map.of("foo", "bar", "number", "12345"));+ CollectionAdminRequest.backupCollection(backupCollectionName, backupName)+ .setLocation(backupLocation)+ .setExtraProperties(extraProps)+ .setRepositoryName(BACKUP_REPO_NAME)+ .processAndWait(cluster.getSolrClient(), 100);+ CollectionAdminResponse response =+ CollectionAdminRequest.listBackup(backupName)+ .setBackupLocation(backupLocation)+ .setBackupRepository(BACKUP_REPO_NAME)+ .process(cluster.getSolrClient());+ assertNotNull(response.getResponse().get("backups"));+ assertTrue(response.getResponse().get("backups") instanceof List);+ List<Map<String, Object>> backups =+ (List<Map<String, Object>>) response.getResponse().get("backups");+ assertEquals(1, backups.size());+ Map<String, Object> backup0 = backups.get(0);+ assertNotNull(backup0.get("extraProperties"));+ assertTrue(backup0.get("extraProperties") instanceof Map);+ Map<String, Object> extraProperties = (Map<String, Object>) backup0.get("extraProperties");+ assertEquals("bar", extraProperties.get("foo"));+ assertEquals("12345", extraProperties.get("number"));++ CollectionAdminRequest.restoreCollection(restoreCollectionName, backupName)+ .setLocation(backupLocation)+ .setRepositoryName(BACKUP_REPO_NAME)+ .processAndWait(solrClient, 500);++ AbstractDistribZkTestBase.waitForRecoveriesToFinish(+ restoreCollectionName, ZkStateReader.from(solrClient), false, false, 3);+ assertEquals(+ numDocs,+ cluster+ .getSolrClient()+ .query(restoreCollectionName, new SolrQuery("*:*"))+ .getResults()+ .getNumFound());+ }+ }}diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/PurgeGraphTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/PurgeGraphTest.javaindex 68599125f38..b2374f89a57 100644--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/PurgeGraphTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/PurgeGraphTest.java@@ -199,7 +199,11 @@ public class PurgeGraphTest extends SolrTestCaseJ4 {private void createBackupIdFile(int backupId, String... shardNames) throws Exception {final BackupProperties createdProps =BackupProperties.create(- "someBackupName", "someCollectionName", "someExtCollectionName", "someConfigName");+ "someBackupName",+ "someCollectionName",+ "someExtCollectionName",+ "someConfigName",+ null);for (String shardName : shardNames) {createdProps.putAndGetShardBackupIdFor(shardName, backupId);}diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adocindex 12137859df3..107b4849209 100644--- a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc@@ -1668,11 +1668,19 @@ This parameter has no effect if `incremental=false` is specified.+[%autowidth,frame=none]|===-|Optional |Default: true+|Optional |Default: `true`|===+Indicates if configset files should be included with the index backup or not. Note that in order to restore a collection, the configset must either exist in ZooKeeper or be part of the backup. Only set this to `false` if you can restore configsets by other means external to Solr (i.e. you have it stored with your application source code, is part of your ZooKeeper backups, etc).+`property.<propertyName>` (V1), `extraProperties` (V2)::+[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++Allows storing additional key/value pairs for custom information related to the backup. In v2, the value is a map of key-value pairs.+`incremental`::+[%autowidth,frame=none]diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.javaindex c7814aab55b..db3ad4543c6 100644--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java@@ -1111,6 +1111,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>protected boolean incremental = true;protected Optional<Integer> maxNumBackupPoints = Optional.empty();protected boolean backupConfigset = true;+ protected Properties extraProperties;public Backup(String collection, String name) {super(CollectionAction.BACKUP, collection);@@ -1205,9 +1206,17 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>return this;}+ public Backup setExtraProperties(Properties extraProperties) {+ this.extraProperties = extraProperties;+ return this;+ }+@Overridepublic SolrParams getParams() {ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();+ if (extraProperties != null) {+ addProperties(params, extraProperties);+ }params.set(CoreAdminParams.COLLECTION, collection);params.set(CoreAdminParams.NAME, name);params.set(CoreAdminParams.BACKUP_LOCATION, location); // note: optionaldiff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.javaindex 3e8a8d9dbae..03acadab7ff 100644--- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java+++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java@@ -21,8 +21,11 @@ import static org.apache.solr.core.TrackingBackupRepository.copiedFiles;import java.io.IOException;import java.io.OutputStream;+import java.io.OutputStreamWriter;+import java.io.Writer;import java.lang.invoke.MethodHandles;import java.net.URI;+import java.nio.charset.StandardCharsets;import java.nio.file.Files;import java.nio.file.Path;import java.util.ArrayList;@@ -456,6 +459,34 @@ public abstract class AbstractIncrementalBackupTest extends SolrCloudTestCase {}}+ public void testBackupProperties() throws IOException {+ BackupProperties p =+ BackupProperties.create(+ "backupName",+ "collection1",+ "collection1-ext",+ "conf1",+ Map.of("foo", "bar", "aaa", "bbb"));+ try (BackupRepository repository =+ cluster.getJettySolrRunner(0).getCoreContainer().newBackupRepository(BACKUP_REPO_NAME)) {+ String backupLocation = repository.getBackupLocation(getBackupLocation());+ URI dest = repository.resolve(repository.createURI(backupLocation), "props-file.properties");+ try (Writer propsWriter =+ new OutputStreamWriter(repository.createOutput(dest), StandardCharsets.UTF_8)) {+ p.store(propsWriter);+ }+ BackupProperties propsRead =+ BackupProperties.readFrom(+ repository, repository.createURI(backupLocation), "props-file.properties");+ assertEquals(p.getCollection(), propsRead.getCollection());+ assertEquals(p.getCollectionAlias(), propsRead.getCollectionAlias());+ assertEquals(p.getConfigName(), propsRead.getConfigName());+ assertEquals(p.getIndexVersion(), propsRead.getIndexVersion());+ assertEquals(p.getExtraProperties(), propsRead.getExtraProperties());+ assertEquals(p.getBackupName(), propsRead.getBackupName());+ }+ }+protected void corruptIndexFiles() throws IOException {List<Slice> slices = new ArrayList<>(getCollectionState(getCollectionName()).getSlices());Replica leader = slices.get(random().nextInt(slices.size())).getLeader();@@ -550,7 +581,7 @@ public abstract class AbstractIncrementalBackupTest extends SolrCloudTestCase {log.info("Indexed {} docs to collection: {}", numDocs, collectionName);}- private int indexDocs(String collectionName, boolean useUUID) throws Exception {+ protected int indexDocs(String collectionName, boolean useUUID) throws Exception {Random random =new Random(docsSeed); // use a constant seed for the whole test run so that we can easily re-index.