Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-15842: Fix async backup response (#479)

ijioio05942bf6c80e
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index befdead1bb5..01dd22cd298 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -558,6 +558,8 @@ Bug Fixes
* SOLR-15944: The Tagger's JSON response format now always uses an object/map to represent each tag instead of an
array, which didn't make sense. (David Smiley)
+* SOLR-15842: Async response for backups now correctly aggregates and returns information (Houston Putman, ijioio)
+
================== 8.11.2 ==================
Bug Fixes
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java
index 3a42a126226..6df02b28381 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java
@@ -45,11 +45,14 @@ import org.apache.solr.handler.component.ShardHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.annotation.Nullable;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.URI;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.List;
import java.util.Optional;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
@@ -255,30 +258,66 @@ public class BackupCmd implements CollApiCmds.CollectionApiCommand {
}
//Aggregating result from different shards
- NamedList<Object> aggRsp = aggregateResults(results, collectionName, backupManager, backupProperties, slices);
+ NamedList<Object> aggRsp = aggregateResults(results, collectionName, slices, backupManager, backupProperties);
results.add("response", aggRsp);
}
- private NamedList<Object> aggregateResults(NamedList<Object> results, String collectionName,
- BackupManager backupManager,
- BackupProperties backupProps,
- Collection<Slice> slices) {
+ private NamedList<Object> aggregateResults(NamedList<Object> results,
+ String collectionName,
+ Collection<Slice> slices,
+ @Nullable BackupManager backupManager,
+ @Nullable BackupProperties backupProps) {
NamedList<Object> aggRsp = new SimpleOrderedMap<>();
aggRsp.add("collection", collectionName);
aggRsp.add("numShards", slices.size());
- aggRsp.add("backupId", backupManager.getBackupId().id);
- aggRsp.add("indexVersion", backupProps.getIndexVersion());
- aggRsp.add("startTime", backupProps.getStartTime());
+ if (backupManager != null) {
+ aggRsp.add("backupId", backupManager.getBackupId().id);
+ }
+ if (backupProps != null) {
+ aggRsp.add("indexVersion", backupProps.getIndexVersion());
+ aggRsp.add("startTime", backupProps.getStartTime());
+ }
- double indexSizeMB = 0;
+ // Optional options for incremental backups
+ Optional<Integer> indexFileCount = Optional.empty();
+ Optional<Integer> uploadedIndexFileCount = Optional.empty();
+ Optional<Double> indexSizeMB = Optional.empty();
+ Optional<Double> uploadedIndexFileMB = Optional.empty();
NamedList<?> shards = (NamedList<?>) results.get("success");
+ List<String> shardBackupIds = new ArrayList<>(shards.size());
for (int i = 0; i < shards.size(); i++) {
NamedList<?> shardResp = (NamedList<?>)((NamedList<?>)shards.getVal(i)).get("response");
if (shardResp == null)
continue;
- indexSizeMB += (double) shardResp.get("indexSizeMB");
+ Integer shardIndexFileCount = (Integer) shardResp.get("indexFileCount");
+ if (shardIndexFileCount != null) {
+ indexFileCount = Optional.of(indexFileCount.orElse(0) + shardIndexFileCount);
+ }
+ Integer shardUploadedIndexFileCount = (Integer) shardResp.get("uploadedIndexFileCount");
+ if (shardUploadedIndexFileCount != null) {
+ uploadedIndexFileCount = Optional.of(uploadedIndexFileCount.orElse(0) + shardUploadedIndexFileCount);
+ }
+ Double shardIndexSizeMB = (Double) shardResp.get("indexSizeMB");
+ if (shardUploadedIndexFileCount != null) {
+ indexSizeMB = Optional.of(indexSizeMB.orElse(0.0) + shardIndexSizeMB);
+ }
+ Double shardUploadedIndexFileMB = (Double) shardResp.get("uploadedIndexFileMB");
+ if (shardUploadedIndexFileMB != null) {
+ uploadedIndexFileMB = Optional.of(uploadedIndexFileMB.orElse(0.0) + shardUploadedIndexFileMB);
+ }
+ Optional.ofNullable((String) shardResp.get("shardBackupId")).ifPresent(shardBackupIds::add);
+ }
+ if (backupProps != null) {
+ backupProps.countIndexFiles(indexFileCount.orElse(0), indexSizeMB.orElse(0.0));
}
- aggRsp.add("indexSizeMB", indexSizeMB);
+ indexFileCount.ifPresent(val -> aggRsp.add("indexFileCount", val));
+ uploadedIndexFileCount.ifPresent(val -> aggRsp.add("uploadedIndexFileCount", val));
+ indexSizeMB.ifPresent(val -> aggRsp.add("indexSizeMB", val));
+ uploadedIndexFileMB.ifPresent(val -> aggRsp.add("uploadedIndexFileMB", val));
+ if (!shardBackupIds.isEmpty()) {
+ aggRsp.add("shardBackupIds", shardBackupIds);
+ }
+
return aggRsp;
}
@@ -323,7 +362,8 @@ public class BackupCmd implements CollApiCmds.CollectionApiCommand {
}
final ShardRequestTracker shardRequestTracker = CollectionHandlingUtils.asyncRequestTracker(asyncId, ccc);
- for (Slice slice : ccc.getZkStateReader().getClusterState().getCollection(collectionName).getActiveSlices()) {
+ Collection<Slice> slices = ccc.getZkStateReader().getClusterState().getCollection(collectionName).getActiveSlices();
+ for (Slice slice : slices) {
Replica replica = null;
if (snapshotMeta.isPresent()) {
@@ -354,5 +394,9 @@ public class BackupCmd implements CollApiCmds.CollectionApiCommand {
log.debug("Sent backup requests to all shard leaders for backupName={}", backupName);
shardRequestTracker.processResponses(results, shardHandler, true, "Could not backup all shards");
+
+ //Aggregating result from different shards
+ NamedList<Object> aggRsp = aggregateResults(results, collectionName, slices, null, null);
+ results.add("response", aggRsp);
}
}
diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
index 6f919d05768..7b095f8e586 100644
--- a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
+++ b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
@@ -62,6 +62,7 @@ public class BackupManager {
public static final String BACKUP_NAME_PROP = "backupName";
public static final String INDEX_VERSION_PROP = "indexVersion";
public static final String START_TIME_PROP = "startTime";
+ public static final String END_TIME_PROP = "endTime";
protected final ZkStateReader zkStateReader;
protected final BackupRepository repository;
diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupProperties.java b/solr/core/src/java/org/apache/solr/core/backup/BackupProperties.java
index 9ea2c7e8cc5..3a4fa6fb760 100644
--- a/solr/core/src/java/org/apache/solr/core/backup/BackupProperties.java
+++ b/solr/core/src/java/org/apache/solr/core/backup/BackupProperties.java
@@ -125,6 +125,7 @@ public class BackupProperties {
public void store(Writer propsWriter) throws IOException {
properties.put("indexSizeMB", String.valueOf(indexSizeMB));
properties.put("indexFileCount", String.valueOf(indexFileCount));
+ properties.put(BackupManager.END_TIME_PROP, Instant.now().toString());
properties.store(propsWriter, "Backup properties file");
}
@@ -145,6 +146,10 @@ public class BackupProperties {
return properties.getProperty(BackupManager.START_TIME_PROP);
}
+ public String getEndTime() {
+ return properties.getProperty(BackupManager.END_TIME_PROP);
+ }
+
public String getIndexVersion() {
return properties.getProperty(BackupManager.INDEX_VERSION_PROP);
}
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java b/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java
index 0dfc6e1500f..3129b60f227 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java
@@ -82,7 +82,7 @@ class BackupCoreOp implements CoreAdminHandler.CoreAdminOp {
"requires a shared file system mounted at the same path on all nodes!");
}
snapShooter.validateCreateSnapshot();
- snapShooter.createSnapshot();
+ it.rsp.addResponse(snapShooter.createSnapshot());
}
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
index f7438dd500e..52b2944aa29 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
@@ -89,9 +89,9 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNa
public static String RUNNING = "running";
public static String COMPLETED = "completed";
public static String FAILED = "failed";
- public static String RESPONSE = "Response";
public static String RESPONSE_STATUS = "STATUS";
public static String RESPONSE_MESSAGE = "msg";
+ public static String OPERATION_RESPONSE = "response";
public CoreAdminHandler() {
super();
@@ -198,6 +198,7 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNa
try {
callInfo.call();
taskObject.setRspObject(callInfo.rsp);
+ taskObject.setOperationRspObject(callInfo.rsp);
} catch (Exception e) {
exceptionCaught = true;
taskObject.setRspObjectFromException(e);
@@ -325,6 +326,7 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNa
static class TaskObject {
String taskId;
String rspInfo;
+ Object operationRspInfo;
public TaskObject(String taskId) {
this.taskId = taskId;
@@ -341,6 +343,14 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNa
public void setRspObjectFromException(Exception e) {
this.rspInfo = e.getMessage();
}
+
+ public Object getOperationRspObject() {
+ return operationRspInfo;
+ }
+
+ public void setOperationRspObject(SolrQueryResponse rspObject) {
+ this.operationRspInfo = rspObject.getResponse();
+ }
}
/**
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
index 7ac25a2ee30..c0b260a73c1 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
@@ -56,7 +56,7 @@ import static org.apache.solr.common.params.CoreAdminParams.SHARD;
import static org.apache.solr.handler.admin.CoreAdminHandler.COMPLETED;
import static org.apache.solr.handler.admin.CoreAdminHandler.CallInfo;
import static org.apache.solr.handler.admin.CoreAdminHandler.FAILED;
-import static org.apache.solr.handler.admin.CoreAdminHandler.RESPONSE;
+import static org.apache.solr.handler.admin.CoreAdminHandler.OPERATION_RESPONSE;
import static org.apache.solr.handler.admin.CoreAdminHandler.RESPONSE_MESSAGE;
import static org.apache.solr.handler.admin.CoreAdminHandler.RESPONSE_STATUS;
import static org.apache.solr.handler.admin.CoreAdminHandler.RUNNING;
@@ -185,10 +185,11 @@ public enum CoreAdminOperation implements CoreAdminOp {
it.rsp.add(RESPONSE_STATUS, RUNNING);
} else if (it.handler.getRequestStatusMap(COMPLETED).containsKey(requestId)) {
it.rsp.add(RESPONSE_STATUS, COMPLETED);
- it.rsp.add(RESPONSE, it.handler.getRequestStatusMap(COMPLETED).get(requestId).getRspObject());
+ it.rsp.add(RESPONSE_MESSAGE, it.handler.getRequestStatusMap(COMPLETED).get(requestId).getRspObject());
+ it.rsp.add(OPERATION_RESPONSE, it.handler.getRequestStatusMap(COMPLETED).get(requestId).getOperationRspObject());
} else if (it.handler.getRequestStatusMap(FAILED).containsKey(requestId)) {
it.rsp.add(RESPONSE_STATUS, FAILED);
- it.rsp.add(RESPONSE, it.handler.getRequestStatusMap(FAILED).get(requestId).getRspObject());
+ it.rsp.add(RESPONSE_MESSAGE, it.handler.getRequestStatusMap(FAILED).get(requestId).getRspObject());
} else {
it.rsp.add(RESPONSE_STATUS, "notfound");
it.rsp.add(RESPONSE_MESSAGE, "No task found in running, completed or failed tasks");
diff --git a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
index b81e46c3531..6e8af649696 100644
--- a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
+++ b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
@@ -100,6 +100,13 @@ All the usages are replaced by BaseHttpSolrClient.RemoteSolrException and BaseHt
* SOLR-15944: The Tagger's JSON response format now always uses an object/map to represent each tag instead of an array.
+* SOLR-15842: Async responses for backups now correctly aggregate and return information.
+In previous versions there was a field returned in async backup status responses, `Response`. This has now been renamed to `msg`, to better fit other collections API responses.
+The `response` field is now a map, containing information about the backup (`startTime`, `indexSizeMB`, `indexFileCount`, etc.).
+
+* SOLR-15884: In Backup request responses, the `response` key now uses a map to return information instead of a list.
+This is only applicable for users returning information in JSON format, which is the default behavior.
+
== New Features & Enhancements
* Replica placement plugins
@@ -145,9 +152,6 @@ Currently this change should only effect compatibility of custom code overriding
* SOLR-14510: The `writeStartDocumentList` in `TextResponseWriter` now receives an extra boolean parameter representing the "exactness" of the `numFound` value (exact vs approximation).
Any custom response writer extending `TextResponseWriter` will need to implement this abstract method now (instead previous with the same name but without the new boolean parameter).
-* SOLR-15884: In Backup request responses, the `response` key now uses a map to return information instead of a list.
-This is only applicable for users returning information in JSON format, which is the default behavior.
-
=== solr.xml maxBooleanClauses now enforced recursively
Lucene 9.0 has additional safety checks over previous versions that impact how the `solr.xml` global `<<configuring-solr-xml#global-maxbooleanclauses,maxBooleanClauses>>` option is enforced.