Apache Solr open-source search software
SOLR-15842: Fix async backup response (#479)
ijioio05942bf6c80e
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex befdead1bb5..01dd22cd298 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -558,6 +558,8 @@ Bug Fixes* SOLR-15944: The Tagger's JSON response format now always uses an object/map to represent each tag instead of anarray, which didn't make sense. (David Smiley)+* SOLR-15842: Async response for backups now correctly aggregates and returns information (Houston Putman, ijioio)+================== 8.11.2 ==================Bug Fixesdiff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.javaindex 3a42a126226..6df02b28381 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java@@ -45,11 +45,14 @@ import org.apache.solr.handler.component.ShardHandler;import org.slf4j.Logger;import org.slf4j.LoggerFactory;+import javax.annotation.Nullable;import java.io.IOException;import java.lang.invoke.MethodHandles;import java.net.URI;+import java.util.ArrayList;import java.util.Collection;import java.util.Collections;+import java.util.List;import java.util.Optional;import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;@@ -255,30 +258,66 @@ public class BackupCmd implements CollApiCmds.CollectionApiCommand {}//Aggregating result from different shards- NamedList<Object> aggRsp = aggregateResults(results, collectionName, backupManager, backupProperties, slices);+ NamedList<Object> aggRsp = aggregateResults(results, collectionName, slices, backupManager, backupProperties);results.add("response", aggRsp);}- private NamedList<Object> aggregateResults(NamedList<Object> results, String collectionName,- BackupManager backupManager,- BackupProperties backupProps,- Collection<Slice> slices) {+ private NamedList<Object> aggregateResults(NamedList<Object> results,+ String collectionName,+ Collection<Slice> slices,+ @Nullable BackupManager backupManager,+ @Nullable BackupProperties backupProps) {NamedList<Object> aggRsp = new SimpleOrderedMap<>();aggRsp.add("collection", collectionName);aggRsp.add("numShards", slices.size());- aggRsp.add("backupId", backupManager.getBackupId().id);- aggRsp.add("indexVersion", backupProps.getIndexVersion());- aggRsp.add("startTime", backupProps.getStartTime());+ if (backupManager != null) {+ aggRsp.add("backupId", backupManager.getBackupId().id);+ }+ if (backupProps != null) {+ aggRsp.add("indexVersion", backupProps.getIndexVersion());+ aggRsp.add("startTime", backupProps.getStartTime());+ }- double indexSizeMB = 0;+ // Optional options for incremental backups+ Optional<Integer> indexFileCount = Optional.empty();+ Optional<Integer> uploadedIndexFileCount = Optional.empty();+ Optional<Double> indexSizeMB = Optional.empty();+ Optional<Double> uploadedIndexFileMB = Optional.empty();NamedList<?> shards = (NamedList<?>) results.get("success");+ List<String> shardBackupIds = new ArrayList<>(shards.size());for (int i = 0; i < shards.size(); i++) {NamedList<?> shardResp = (NamedList<?>)((NamedList<?>)shards.getVal(i)).get("response");if (shardResp == null)continue;- indexSizeMB += (double) shardResp.get("indexSizeMB");+ Integer shardIndexFileCount = (Integer) shardResp.get("indexFileCount");+ if (shardIndexFileCount != null) {+ indexFileCount = Optional.of(indexFileCount.orElse(0) + shardIndexFileCount);+ }+ Integer shardUploadedIndexFileCount = (Integer) shardResp.get("uploadedIndexFileCount");+ if (shardUploadedIndexFileCount != null) {+ uploadedIndexFileCount = Optional.of(uploadedIndexFileCount.orElse(0) + shardUploadedIndexFileCount);+ }+ Double shardIndexSizeMB = (Double) shardResp.get("indexSizeMB");+ if (shardUploadedIndexFileCount != null) {+ indexSizeMB = Optional.of(indexSizeMB.orElse(0.0) + shardIndexSizeMB);+ }+ Double shardUploadedIndexFileMB = (Double) shardResp.get("uploadedIndexFileMB");+ if (shardUploadedIndexFileMB != null) {+ uploadedIndexFileMB = Optional.of(uploadedIndexFileMB.orElse(0.0) + shardUploadedIndexFileMB);+ }+ Optional.ofNullable((String) shardResp.get("shardBackupId")).ifPresent(shardBackupIds::add);+ }+ if (backupProps != null) {+ backupProps.countIndexFiles(indexFileCount.orElse(0), indexSizeMB.orElse(0.0));}- aggRsp.add("indexSizeMB", indexSizeMB);+ indexFileCount.ifPresent(val -> aggRsp.add("indexFileCount", val));+ uploadedIndexFileCount.ifPresent(val -> aggRsp.add("uploadedIndexFileCount", val));+ indexSizeMB.ifPresent(val -> aggRsp.add("indexSizeMB", val));+ uploadedIndexFileMB.ifPresent(val -> aggRsp.add("uploadedIndexFileMB", val));+ if (!shardBackupIds.isEmpty()) {+ aggRsp.add("shardBackupIds", shardBackupIds);+ }+return aggRsp;}@@ -323,7 +362,8 @@ public class BackupCmd implements CollApiCmds.CollectionApiCommand {}final ShardRequestTracker shardRequestTracker = CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc);- for (Slice slice : ccc.getZkStateReader().getClusterState().getCollection(collectionName).getActiveSlices()) {+ Collection<Slice> slices = ccc.getZkStateReader().getClusterState().getCollection(collectionName).getActiveSlices();+ for (Slice slice : slices) {Replica replica = null;if (snapshotMeta.isPresent()) {@@ -354,5 +394,9 @@ public class BackupCmd implements CollApiCmds.CollectionApiCommand {log.debug("Sent backup requests to all shard leaders for backupName={}", backupName);shardRequestTracker.processResponses(results, shardHandler, true, "Could not backup all shards");++ //Aggregating result from different shards+ NamedList<Object> aggRsp = aggregateResults(results, collectionName, slices, null, null);+ results.add("response", aggRsp);}}diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.javaindex 6f919d05768..7b095f8e586 100644--- a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java+++ b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java@@ -62,6 +62,7 @@ public class BackupManager {public static final String BACKUP_NAME_PROP = "backupName";public static final String INDEX_VERSION_PROP = "indexVersion";public static final String START_TIME_PROP = "startTime";+ public static final String END_TIME_PROP = "endTime";protected final ZkStateReader zkStateReader;protected final BackupRepository repository;diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupProperties.java b/solr/core/src/java/org/apache/solr/core/backup/BackupProperties.javaindex 9ea2c7e8cc5..3a4fa6fb760 100644--- a/solr/core/src/java/org/apache/solr/core/backup/BackupProperties.java+++ b/solr/core/src/java/org/apache/solr/core/backup/BackupProperties.java@@ -125,6 +125,7 @@ public class BackupProperties {public void store(Writer propsWriter) throws IOException {properties.put("indexSizeMB", String.valueOf(indexSizeMB));properties.put("indexFileCount", String.valueOf(indexFileCount));+ properties.put(BackupManager.END_TIME_PROP, Instant.now().toString());properties.store(propsWriter, "Backup properties file");}@@ -145,6 +146,10 @@ public class BackupProperties {return properties.getProperty(BackupManager.START_TIME_PROP);}+ public String getEndTime() {+ return properties.getProperty(BackupManager.END_TIME_PROP);+ }+public String getIndexVersion() {return properties.getProperty(BackupManager.INDEX_VERSION_PROP);}diff --git a/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java b/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.javaindex 0dfc6e1500f..3129b60f227 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java@@ -82,7 +82,7 @@ class BackupCoreOp implements CoreAdminHandler.CoreAdminOp {"requires a shared file system mounted at the same path on all nodes!");}snapShooter.validateCreateSnapshot();- snapShooter.createSnapshot();+ it.rsp.addResponse(snapShooter.createSnapshot());}} catch (Exception e) {throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.javaindex f7438dd500e..52b2944aa29 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java@@ -89,9 +89,9 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNapublic static String RUNNING = "running";public static String COMPLETED = "completed";public static String FAILED = "failed";- public static String RESPONSE = "Response";public static String RESPONSE_STATUS = "STATUS";public static String RESPONSE_MESSAGE = "msg";+ public static String OPERATION_RESPONSE = "response";public CoreAdminHandler() {super();@@ -198,6 +198,7 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNatry {callInfo.call();taskObject.setRspObject(callInfo.rsp);+ taskObject.setOperationRspObject(callInfo.rsp);} catch (Exception e) {exceptionCaught = true;taskObject.setRspObjectFromException(e);@@ -325,6 +326,7 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNastatic class TaskObject {String taskId;String rspInfo;+ Object operationRspInfo;public TaskObject(String taskId) {this.taskId = taskId;@@ -341,6 +343,14 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNapublic void setRspObjectFromException(Exception e) {this.rspInfo = e.getMessage();}++ public Object getOperationRspObject() {+ return operationRspInfo;+ }++ public void setOperationRspObject(SolrQueryResponse rspObject) {+ this.operationRspInfo = rspObject.getResponse();+ }}/**diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.javaindex 7ac25a2ee30..c0b260a73c1 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java@@ -56,7 +56,7 @@ import static org.apache.solr.common.params.CoreAdminParams.SHARD;import static org.apache.solr.handler.admin.CoreAdminHandler.COMPLETED;import static org.apache.solr.handler.admin.CoreAdminHandler.CallInfo;import static org.apache.solr.handler.admin.CoreAdminHandler.FAILED;-import static org.apache.solr.handler.admin.CoreAdminHandler.RESPONSE;+import static org.apache.solr.handler.admin.CoreAdminHandler.OPERATION_RESPONSE;import static org.apache.solr.handler.admin.CoreAdminHandler.RESPONSE_MESSAGE;import static org.apache.solr.handler.admin.CoreAdminHandler.RESPONSE_STATUS;import static org.apache.solr.handler.admin.CoreAdminHandler.RUNNING;@@ -185,10 +185,11 @@ public enum CoreAdminOperation implements CoreAdminOp {it.rsp.add(RESPONSE_STATUS, RUNNING);} else if (it.handler.getRequestStatusMap(COMPLETED).containsKey(requestId)) {it.rsp.add(RESPONSE_STATUS, COMPLETED);- it.rsp.add(RESPONSE, it.handler.getRequestStatusMap(COMPLETED).get(requestId).getRspObject());+ it.rsp.add(RESPONSE_MESSAGE, it.handler.getRequestStatusMap(COMPLETED).get(requestId).getRspObject());+ it.rsp.add(OPERATION_RESPONSE, it.handler.getRequestStatusMap(COMPLETED).get(requestId).getOperationRspObject());} else if (it.handler.getRequestStatusMap(FAILED).containsKey(requestId)) {it.rsp.add(RESPONSE_STATUS, FAILED);- it.rsp.add(RESPONSE, it.handler.getRequestStatusMap(FAILED).get(requestId).getRspObject());+ it.rsp.add(RESPONSE_MESSAGE, it.handler.getRequestStatusMap(FAILED).get(requestId).getRspObject());} else {it.rsp.add(RESPONSE_STATUS, "notfound");it.rsp.add(RESPONSE_MESSAGE, "No task found in running, completed or failed tasks");diff --git a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/src/major-changes-in-solr-9.adocindex b81e46c3531..6e8af649696 100644--- a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc+++ b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc@@ -100,6 +100,13 @@ All the usages are replaced by BaseHttpSolrClient.RemoteSolrException and BaseHt* SOLR-15944: The Tagger's JSON response format now always uses an object/map to represent each tag instead of an array.+* SOLR-15842: Async responses for backups now correctly aggregate and return information.+In previous versions there was a field returned in async backup status responses, `Response`. This has now been renamed to `msg`, to better fit other collections API responses.+The `response` field is now a map, containing information about the backup (`startTime`, `indexSizeMB`, `indexFileCount`, etc.).++* SOLR-15884: In Backup request responses, the `response` key now uses a map to return information instead of a list.+This is only applicable for users returning information in JSON format, which is the default behavior.+== New Features & Enhancements* Replica placement plugins@@ -145,9 +152,6 @@ Currently this change should only effect compatibility of custom code overriding* SOLR-14510: The `writeStartDocumentList` in `TextResponseWriter` now receives an extra boolean parameter representing the "exactness" of the `numFound` value (exact vs approximation).Any custom response writer extending `TextResponseWriter` will need to implement this abstract method now (instead previous with the same name but without the new boolean parameter).-* SOLR-15884: In Backup request responses, the `response` key now uses a map to return information instead of a list.-This is only applicable for users returning information in JSON format, which is the default behavior.-=== solr.xml maxBooleanClauses now enforced recursivelyLucene 9.0 has additional safety checks over previous versions that impact how the `solr.xml` global `<<configuring-solr-xml#global-maxbooleanclauses,maxBooleanClauses>>` option is enforced.