Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-15702: Stabilize S3 tests by using waiters (#356)

Houston Putman97b0df77a5c0
diff --git a/solr/contrib/s3-repository/src/java/org/apache/solr/s3/S3StorageClient.java b/solr/contrib/s3-repository/src/java/org/apache/solr/s3/S3StorageClient.java
index 3621136386b..fd818b78ef2 100644
--- a/solr/contrib/s3-repository/src/java/org/apache/solr/s3/S3StorageClient.java
+++ b/solr/contrib/s3-repository/src/java/org/apache/solr/s3/S3StorageClient.java
@@ -23,6 +23,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.lang.invoke.MethodHandles;
import java.net.URI;
+import java.time.Duration;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashSet;
@@ -37,6 +38,7 @@ import org.slf4j.LoggerFactory;
import software.amazon.awssdk.awscore.exception.AwsServiceException;
import software.amazon.awssdk.core.exception.SdkClientException;
import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.core.retry.backoff.BackoffStrategy;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.http.apache.ApacheHttpClient;
import software.amazon.awssdk.http.apache.ProxyConfiguration;
@@ -129,10 +131,10 @@ public class S3StorageClient {
/** Create a directory in S3. */
void createDirectory(String path) throws S3Exception {
- path = sanitizedDirPath(path);
+ String sanitizedDirPath = sanitizedDirPath(path);
- if (!parentDirectoryExist(path)) {
- createDirectory(getParentDirectory(path));
+ if (!parentDirectoryExist(sanitizedDirPath)) {
+ createDirectory(getParentDirectory(sanitizedDirPath));
// TODO see https://issues.apache.org/jira/browse/SOLR-15359
// throw new S3Exception("Parent directory doesn't exist, path=" + path);
}
@@ -143,9 +145,18 @@ public class S3StorageClient {
PutObjectRequest.builder()
.bucket(bucketName)
.contentType(S3_DIR_CONTENT_TYPE)
- .key(path)
+ .key(sanitizedDirPath)
.build();
s3Client.putObject(putRequest, RequestBody.empty());
+ // Wait until the object exists to continue
+ s3Client
+ .waiter()
+ .waitUntilObjectExists(
+ builder -> builder.bucket(bucketName).key(sanitizedDirPath),
+ config ->
+ config
+ .waitTimeout(Duration.ofSeconds(5))
+ .backoffStrategy(BackoffStrategy.defaultStrategy()));
} catch (SdkClientException ase) {
throw handleAmazonException(ase);
}
@@ -446,11 +457,7 @@ public class S3StorageClient {
}
// Check for existence twice, because s3Mock has issues in the tests
- if (pathExists(parentDirectory)) {
- return true;
- } else {
- return pathExists(parentDirectory);
- }
+ return pathExists(parentDirectory);
}
private String getParentDirectory(String path) {