Apache Solr open-source search software
SOLR-15982: Add end time value to backup response (#595)
ijioio958a070c9280
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex bea6cf74f5b..30850b028c7 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -301,6 +301,8 @@ when told to. The admin UI now tells it to. (Nazerke Seidan, David Smiley)* SOLR-15965: Use better signatures for the PKI Authentication plugin. (Mike Drob)+* SOLR-15982: Add end time value to backup response, standardize backup response key names and date formats (Artem Abeleshev, Christine Poerschke, Houston Putman)+Build---------------------* LUCENE-9077 LUCENE-9433: Support Gradle build, remove Ant support from trunk (Dawid Weiss, Erick Erickson, Uwe Schindler et.al.)diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.javaindex 7c92e540b33..7aab96431f8 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java@@ -111,11 +111,15 @@ public class BackupCmd implements CollApiCmds.CollectionApiCommand {throw e;}} else {- copyIndexFiles(backupUri, collectionName, message, results);+ copyIndexFiles(backupUri, collectionName, message, results, backupProperties, backupMgr);}break;}case CollectionAdminParams.NO_INDEX_BACKUP_STRATEGY: {+ NamedList<Object> response = new SimpleOrderedMap<>();+ response.add("collection", collectionName);+ response.add("startTime", backupProperties.getStartTime());+ results.add("response", response);break;}}@@ -137,6 +141,13 @@ public class BackupCmd implements CollApiCmds.CollectionApiCommand {backupMgr.writeBackupProperties(backupProperties);+ // It is safe to extract results here+ @SuppressWarnings("unchecked")+ NamedList<Object> response = (NamedList<Object>) results.get("response");+ // Note that endTime can't be handled within aggregateResults call+ // since it is defined later+ response.add("endTime", backupProperties.getEndTime());+log.info("Completed backing up ZK data for backupName={}", backupName);int maxNumBackup = message.getInt(CoreAdminParams.MAX_NUM_BACKUP_POINTS, -1);@@ -257,7 +268,7 @@ public class BackupCmd implements CollApiCmds.CollectionApiCommand {}//Aggregating result from different shards- NamedList<Object> aggRsp = aggregateResults(results, collectionName, slices, backupManager, backupProperties);+ NamedList<Object> aggRsp = aggregateResults(results, collectionName, slices, backupManager, backupProperties, true);results.add("response", aggRsp);}@@ -265,19 +276,18 @@ public class BackupCmd implements CollApiCmds.CollectionApiCommand {String collectionName,Collection<Slice> slices,BackupManager backupManager,- BackupProperties backupProps) {+ BackupProperties backupProps,+ boolean incremental) {NamedList<Object> aggRsp = new SimpleOrderedMap<>();aggRsp.add("collection", collectionName);aggRsp.add("numShards", slices.size());- if (backupManager != null) {+ if(incremental) {aggRsp.add("backupId", backupManager.getBackupId().id);}- if (backupProps != null) {- aggRsp.add("indexVersion", backupProps.getIndexVersion());- aggRsp.add("startTime", backupProps.getStartTime());- }+ aggRsp.add("indexVersion", backupProps.getIndexVersion());+ aggRsp.add("startTime", backupProps.getStartTime());- // Optional options for incremental backups+ // Optional options for backupsOptional<Integer> indexFileCount = Optional.empty();Optional<Integer> uploadedIndexFileCount = Optional.empty();Optional<Double> indexSizeMB = Optional.empty();@@ -297,7 +307,7 @@ public class BackupCmd implements CollApiCmds.CollectionApiCommand {uploadedIndexFileCount = Optional.of(uploadedIndexFileCount.orElse(0) + shardUploadedIndexFileCount);}Double shardIndexSizeMB = (Double) shardResp.get("indexSizeMB");- if (shardUploadedIndexFileCount != null) {+ if (shardIndexSizeMB != null) {indexSizeMB = Optional.of(indexSizeMB.orElse(0.0) + shardIndexSizeMB);}Double shardUploadedIndexFileMB = (Double) shardResp.get("uploadedIndexFileMB");@@ -331,7 +341,8 @@ public class BackupCmd implements CollApiCmds.CollectionApiCommand {return params;}- private void copyIndexFiles(URI backupPath, String collectionName, ZkNodeProps request, NamedList<Object> results) throws Exception {+ private void copyIndexFiles(URI backupPath, String collectionName, ZkNodeProps request, NamedList<Object> results, BackupProperties backupProperties,+ BackupManager backupManager) throws Exception {String backupName = request.getStr(NAME);String asyncId = request.getStr(ASYNC);String repoName = request.getStr(CoreAdminParams.BACKUP_REPOSITORY);@@ -395,7 +406,7 @@ public class BackupCmd implements CollApiCmds.CollectionApiCommand {shardRequestTracker.processResponses(results, shardHandler, true, "Could not backup all shards");//Aggregating result from different shards- NamedList<Object> aggRsp = aggregateResults(results, collectionName, slices, null, null);+ NamedList<Object> aggRsp = aggregateResults(results, collectionName, slices, backupManager, backupProperties, false);results.add("response", aggRsp);}}diff --git a/solr/core/src/java/org/apache/solr/handler/SnapShooter.java b/solr/core/src/java/org/apache/solr/handler/SnapShooter.javaindex 443613fad98..b8850912851 100644--- a/solr/core/src/java/org/apache/solr/handler/SnapShooter.java+++ b/solr/core/src/java/org/apache/solr/handler/SnapShooter.java@@ -21,6 +21,7 @@ import java.lang.invoke.MethodHandles;import java.net.URI;import java.nio.file.Paths;import java.text.SimpleDateFormat;+import java.time.Instant;import java.util.ArrayList;import java.util.Collection;import java.util.Collections;@@ -259,7 +260,7 @@ public class SnapShooter {boolean success = false;try {NamedList<Object> details = new SimpleOrderedMap<>();- details.add("startTime", new Date().toString());//bad; should be Instant.now().toString()+ details.add("startTime", Instant.now().toString());Collection<String> files = indexCommit.getFileNames();Directory dir = solrCore.getDirectoryFactory().get(solrCore.getIndexDir(), DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType);@@ -272,9 +273,13 @@ public class SnapShooter {solrCore.getDirectoryFactory().release(dir);}- details.add("fileCount", files.size());+ String endTime = Instant.now().toString();++ details.add("fileCount", files.size()); // DEPRECATED: for removal, replaced with indexFileCount+ details.add("indexFileCount", files.size());details.add("status", "success");- details.add("snapshotCompletedAt", new Date().toString());//bad; should be Instant.now().toString()+ details.add("snapshotCompletedAt", endTime); // DEPRECATED: for removal, replaced with endTime+ details.add("endTime", endTime);details.add("snapshotName", snapshotName);details.add("directoryName", directoryName);if (log.isInfoEnabled()) {@@ -320,14 +325,18 @@ public class SnapShooter {log.info("Deleting snapshot: {}", snapshotName);NamedList<Object> details = new NamedList<>();+ details.add("startTime", Instant.now().toString());details.add("snapshotName", snapshotName);try {URI path = baseSnapDirPath.resolve("snapshot." + snapshotName);backupRepo.deleteDirectory(path);+ String endTime = Instant.now().toString();+details.add("status", "success");- details.add("snapshotDeletedAt", new Date().toString());+ details.add("snapshotDeletedAt", endTime); // DEPRECATED: for removal, replaced with endTime+ details.add("endTime", endTime);} catch (IOException e) {details.add("status", "Unable to delete snapshot: " + snapshotName);diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adocindex c6eef7ad283..70c62670e28 100644--- a/solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/backup-restore.adoc@@ -140,10 +140,12 @@ http://localhost:8983/solr/gettingstarted/replication?command=details&wt=xml[source,xml]----<lst name="backup">- <str name="startTime">Sun Apr 12 16:22:50 DAVT 2015</str>+ <str name="startTime">2022-02-11T17:19:33.271461700Z</str><int name="fileCount">10</int>+ <int name="indexFileCount">10</int><str name="status">success</str>- <str name="snapshotCompletedAt">Sun Apr 12 16:22:50 DAVT 2015</str>+ <str name="snapshotCompletedAt">2022-02-11T17:19:34.363859100Z</str>+ <str name="endTime">2022-02-11T17:19:34.363859100Z</str><str name="snapshotName">my_backup</str></lst>----diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adocindex 837698272c7..95e0a9b7446 100644--- a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc@@ -1658,6 +1658,34 @@ A boolean parameter allowing users to choose whether to create an incremental (`If unspecified, backups are done incrementally by default.Incremental backups are preferred in all known circumstances and snapshot backups are deprecated, so this parameter should only be used after much consideration.++[example.tab-pane#backup-response-incremental]+====+[.tab-label]*Incremental Backup Response*++[source,json]+----+{+ "response": {+ "collection": "techproducts",+ "numShards": 2,+ "backupId": 0,+ "indexVersion": "9.0.0",+ "startTime": "2022-02-11T17:20:44.157305500Z",+ "indexFileCount": 22,+ "uploadedIndexFileCount": 22,+ "indexSizeMB": 0.007,+ "uploadedIndexFileMB": 0.007,+ "shardBackupIds": [+ "md_shard2_0",+ "md_shard1_0"+ ],+ "endTime": "2022-02-11T17:20:45.245534400Z"+ }+}+----+====+[[listbackup]]== LISTBACKUP: List Backupsdiff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adocindex 28f1cc42a8d..55c15e9f1d8 100644--- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc+++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc@@ -113,6 +113,12 @@ All the usages are replaced by BaseHttpSolrClient.RemoteSolrException and BaseHtIn previous versions there was a field returned in async backup status responses, `Response`. This has now been renamed to `msg`, to better fit other collections API responses.The `response` field is now a map, containing information about the backup (`startTime`, `indexSizeMB`, `indexFileCount`, etc.).+* SOLR-15982: For collection's snapshot backup request responses additional fields `indexVersion`, `indexFileCount`, etc. were added similar to incremental backup request responses.+Also, both snapshot and incremental backup request responses will now contain `starTime` and `endTime`.+Snapshot backup shard's response were updated to add fields `indexFileCount` and `endTime`, snapshot delete shard's response were updated to add fields `startTime` and `endTime`.+Previous fields `fileCount`, `snapshotCompletedAt` and `snapshotDeletedAt` of backup and delete shard's responses are now deprecated and will be removed in future releases.+All date/time fields of backup and delete related shard's responses have been updated to use `Instance` instead of `Date`, meaning the output will be in the standard ISO 8601 Format.+* SOLR-15884: In Backup request responses, the `response` key now uses a map to return information instead of a list.This is only applicable for users returning information in JSON format, which is the default behavior.