Apache Solr open-source search software
SOLR-16026: Enable spotless on gcs-repository module
Kevin Risden8df4a0659293
diff --git a/gradle/validation/spotless.gradle b/gradle/validation/spotless.gradleindex d22229c1518..d93e7f28656 100644--- a/gradle/validation/spotless.gradle+++ b/gradle/validation/spotless.gradle@@ -44,7 +44,6 @@ configure(project(":solr").subprojects) { prj ->// Exclude certain files (generated ones, mostly).switch (project.path) {- case ":solr:modules:gcs-repository":case ":solr:modules:hadoop-auth":case ":solr:modules:hdfs":case ":solr:modules:sql":diff --git a/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/GCSBackupRepository.java b/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/GCSBackupRepository.javaindex fb3b5a49b18..6d110f400c8 100644--- a/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/GCSBackupRepository.java+++ b/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/GCSBackupRepository.java@@ -17,6 +17,8 @@package org.apache.solr.gcs;+import static java.net.HttpURLConnection.HTTP_PRECON_FAILED;+import com.google.auth.oauth2.GoogleCredentials;import com.google.cloud.ReadChannel;import com.google.cloud.WriteChannel;@@ -26,21 +28,6 @@ import com.google.cloud.storage.BlobInfo;import com.google.cloud.storage.Storage;import com.google.cloud.storage.StorageException;import com.google.cloud.storage.StorageOptions;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.index.CorruptIndexException;-import org.apache.lucene.store.BufferedIndexInput;-import org.apache.lucene.store.ChecksumIndexInput;-import org.apache.lucene.store.Directory;-import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IndexInput;-import org.apache.lucene.store.IndexOutput;-import org.apache.solr.common.params.CoreAdminParams;-import org.apache.solr.common.util.NamedList;-import org.apache.solr.core.DirectoryFactory;-import org.apache.solr.core.backup.repository.BackupRepository;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;-import java.io.FileInputStream;import java.io.IOException;import java.io.OutputStream;@@ -58,436 +45,450 @@ import java.util.LinkedList;import java.util.List;import java.util.Objects;import java.util.stream.Collectors;+import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.index.CorruptIndexException;+import org.apache.lucene.store.BufferedIndexInput;+import org.apache.lucene.store.ChecksumIndexInput;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.IndexOutput;+import org.apache.solr.common.params.CoreAdminParams;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.core.DirectoryFactory;+import org.apache.solr.core.backup.repository.BackupRepository;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;-import static java.net.HttpURLConnection.HTTP_PRECON_FAILED;--/**- * {@link BackupRepository} implementation that stores files in Google Cloud Storage ("GCS").- */+/** {@link BackupRepository} implementation that stores files in Google Cloud Storage ("GCS"). */public class GCSBackupRepository implements BackupRepository {- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());- private static final int LARGE_BLOB_THRESHOLD_BYTE_SIZE = 5 * 1024 * 1024;- private static final Storage.BlobWriteOption[] NO_WRITE_OPTIONS = new Storage.BlobWriteOption[0];-- protected Storage storage;-- private NamedList<?> config = null;- protected String bucketName = null;- protected String credentialPath = null;- protected int writeBufferSizeBytes;- protected int readBufferSizeBytes;- protected StorageOptions.Builder storageOptionsBuilder = null;-- protected Storage initStorage() {- if (storage != null)- return storage;-- try {- if (credentialPath != null) {- log.info("Creating GCS client using credential at {}", credentialPath);- // 'GoogleCredentials.fromStream' closes the input stream, so we don't- GoogleCredentials credential = GoogleCredentials.fromStream(new FileInputStream(credentialPath));- storageOptionsBuilder.setCredentials(credential);- } else {- // nowarn compile time string concatenation- log.warn(GCSConfigParser.potentiallyMissingCredentialMsg()); //nowarn- }- storage = storageOptionsBuilder.build().getService();- } catch (IOException e) {- throw new IllegalStateException(e);- }- return storage;+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());+ private static final int LARGE_BLOB_THRESHOLD_BYTE_SIZE = 5 * 1024 * 1024;+ private static final Storage.BlobWriteOption[] NO_WRITE_OPTIONS = new Storage.BlobWriteOption[0];++ protected Storage storage;++ private NamedList<?> config = null;+ protected String bucketName = null;+ protected String credentialPath = null;+ protected int writeBufferSizeBytes;+ protected int readBufferSizeBytes;+ protected StorageOptions.Builder storageOptionsBuilder = null;++ protected Storage initStorage() {+ if (storage != null) return storage;++ try {+ if (credentialPath != null) {+ log.info("Creating GCS client using credential at {}", credentialPath);+ // 'GoogleCredentials.fromStream' closes the input stream, so we don't+ GoogleCredentials credential =+ GoogleCredentials.fromStream(new FileInputStream(credentialPath));+ storageOptionsBuilder.setCredentials(credential);+ } else {+ // nowarn compile time string concatenation+ log.warn(GCSConfigParser.potentiallyMissingCredentialMsg()); // nowarn+ }+ storage = storageOptionsBuilder.build().getService();+ } catch (IOException e) {+ throw new IllegalStateException(e);}-- @Override- public void init(NamedList<?> args) {- this.config = args;- final GCSConfigParser configReader = new GCSConfigParser();- final GCSConfigParser.GCSConfig parsedConfig = configReader.parseConfiguration(config);-- this.bucketName = parsedConfig.getBucketName();- this.credentialPath = parsedConfig.getCredentialPath();- this.writeBufferSizeBytes = parsedConfig.getWriteBufferSize();- this.readBufferSizeBytes = parsedConfig.getReadBufferSize();- this.storageOptionsBuilder = parsedConfig.getStorageOptionsBuilder();-- initStorage();+ return storage;+ }++ @Override+ public void init(NamedList<?> args) {+ this.config = args;+ final GCSConfigParser configReader = new GCSConfigParser();+ final GCSConfigParser.GCSConfig parsedConfig = configReader.parseConfiguration(config);++ this.bucketName = parsedConfig.getBucketName();+ this.credentialPath = parsedConfig.getCredentialPath();+ this.writeBufferSizeBytes = parsedConfig.getWriteBufferSize();+ this.readBufferSizeBytes = parsedConfig.getReadBufferSize();+ this.storageOptionsBuilder = parsedConfig.getStorageOptionsBuilder();++ initStorage();+ }++ @Override+ @SuppressWarnings("unchecked")+ public <T> T getConfigProperty(String name) {+ return (T) this.config.get(name);+ }++ @Override+ public URI createURI(String location) {+ Objects.requireNonNull(location);++ URI result;+ try {+ result = new URI(location);+ } catch (URISyntaxException e) {+ throw new IllegalArgumentException("Error on creating URI", e);}- @Override- @SuppressWarnings("unchecked")- public <T> T getConfigProperty(String name) {- return (T) this.config.get(name);- }-- @Override- public URI createURI(String location) {- Objects.requireNonNull(location);+ return result;+ }- URI result;- try {- result = new URI(location);- } catch (URISyntaxException e) {- throw new IllegalArgumentException("Error on creating URI", e);- }+ @Override+ public URI createDirectoryURI(String location) {+ Objects.requireNonNull(location);- return result;+ if (!location.endsWith("/")) {+ location += "/";}- @Override- public URI createDirectoryURI(String location) {- Objects.requireNonNull(location);+ return createURI(location);+ }- if (!location.endsWith("/")) {- location += "/";+ @Override+ public URI resolve(URI baseUri, String... pathComponents) {+ StringBuilder builder = new StringBuilder(baseUri.toString());+ for (String path : pathComponents) {+ if (path != null && !path.isEmpty()) {+ if (builder.charAt(builder.length() - 1) != '/') {+ builder.append('/');}-- return createURI(location);+ builder.append(path);+ }}- @Override- public URI resolve(URI baseUri, String... pathComponents) {- StringBuilder builder = new StringBuilder(baseUri.toString());- for (String path : pathComponents) {- if (path != null && !path.isEmpty()) {- if (builder.charAt(builder.length()-1) != '/') {- builder.append('/');- }- builder.append(path);- }- }-- return URI.create(builder.toString()).normalize();+ return URI.create(builder.toString()).normalize();+ }++ @Override+ public URI resolveDirectory(URI baseUri, String... pathComponents) {+ if (pathComponents.length > 0) {+ if (!pathComponents[pathComponents.length - 1].endsWith("/")) {+ pathComponents[pathComponents.length - 1] = pathComponents[pathComponents.length - 1] + "/";+ }+ } else {+ if (!baseUri.getPath().endsWith("/")) {+ baseUri = URI.create(baseUri + "/");+ }}+ return resolve(baseUri, pathComponents);+ }- @Override- public URI resolveDirectory(URI baseUri, String... pathComponents) {- if (pathComponents.length > 0) {- if (!pathComponents[pathComponents.length - 1].endsWith("/")) {- pathComponents[pathComponents.length - 1] = pathComponents[pathComponents.length - 1] + "/";- }- } else {- if (!baseUri.getPath().endsWith("/")) {- baseUri = URI.create(baseUri + "/");- }- }- return resolve(baseUri, pathComponents);- }+ @Override+ public boolean exists(URI path) throws IOException {+ return exists(path.toString());+ }- @Override- public boolean exists(URI path) throws IOException {- return exists(path.toString());+ public boolean exists(String path) throws IOException {+ if (path.equals(getConfigProperty(CoreAdminParams.BACKUP_LOCATION))) {+ return true;}- public boolean exists(String path) throws IOException {- if (path.equals(getConfigProperty(CoreAdminParams.BACKUP_LOCATION))) {- return true;- }-- if (path.endsWith("/")) {- return storage.get(bucketName, path, Storage.BlobGetOption.fields()) != null;- } else {- final String filePath = path;- final String directoryPath = path + "/";- return storage.get(bucketName, filePath, Storage.BlobGetOption.fields()) != null ||- storage.get(bucketName, directoryPath, Storage.BlobGetOption.fields()) != null;- }- }-- @Override- public PathType getPathType(URI path) throws IOException {- if (path.toString().endsWith("/"))- return PathType.DIRECTORY;-- Blob blob = storage.get(bucketName, path.toString()+"/", Storage.BlobGetOption.fields());- if (blob != null)- return PathType.DIRECTORY;-- return PathType.FILE;+ if (path.endsWith("/")) {+ return storage.get(bucketName, path, Storage.BlobGetOption.fields()) != null;+ } else {+ final String filePath = path;+ final String directoryPath = path + "/";+ return storage.get(bucketName, filePath, Storage.BlobGetOption.fields()) != null+ || storage.get(bucketName, directoryPath, Storage.BlobGetOption.fields()) != null;}-- @Override- public String[] listAll(URI path) throws IOException {- final String blobName = appendTrailingSeparatorIfNecessary(path.toString());-- final String pathStr = blobName;- final LinkedList<String> result = new LinkedList<>();- storage.list(- bucketName,- Storage.BlobListOption.currentDirectory(),- Storage.BlobListOption.prefix(pathStr),- Storage.BlobListOption.fields())- .iterateAll().forEach(- blob -> {- assert blob.getName().startsWith(pathStr);- final String suffixName = blob.getName().substring(pathStr.length());- if (!suffixName.isEmpty()) {- // Remove trailing '/' if present- if (suffixName.endsWith("/")) {- result.add(suffixName.substring(0, suffixName.length() - 1));- } else {- result.add(suffixName);- }- }- });-- return result.toArray(new String[0]);- }-- @Override- public IndexInput openInput(URI dirPath, String fileName, IOContext ctx) throws IOException {- return openInput(dirPath, fileName, ctx, readBufferSizeBytes);- }-- private IndexInput openInput(URI dirPath, String fileName, IOContext ctx, int bufferSize) {- String blobName = resolve(dirPath, fileName).toString();-- final BlobId blobId = BlobId.of(bucketName, blobName);- final Blob blob = storage.get(blobId, Storage.BlobGetOption.fields(Storage.BlobField.SIZE));- final ReadChannel readChannel = blob.reader();- readChannel.setChunkSize(bufferSize);-- return new BufferedIndexInput(blobName, bufferSize) {-- @Override- public long length() {- return blob.getSize();- }-- @Override- protected void readInternal(ByteBuffer b) throws IOException {- readChannel.read(b);- }-- @Override- protected void seekInternal(long pos) throws IOException {- readChannel.seek(pos);- }-- @Override- public void close() throws IOException {- readChannel.close();- }- };- }-- @Override- public OutputStream createOutput(URI path) throws IOException {- final BlobInfo blobInfo = BlobInfo.newBuilder(bucketName, path.toString()).build();- final WriteChannel writeChannel = storage.writer(blobInfo, getDefaultBlobWriteOptions());-- return Channels.newOutputStream(new WritableByteChannel() {- @Override- public int write(ByteBuffer src) throws IOException {- return writeChannel.write(src);- }-- @Override- public boolean isOpen() {- return writeChannel.isOpen();- }-- @Override- public void close() throws IOException {- writeChannel.close();- }+ }++ @Override+ public PathType getPathType(URI path) throws IOException {+ if (path.toString().endsWith("/")) return PathType.DIRECTORY;++ Blob blob = storage.get(bucketName, path.toString() + "/", Storage.BlobGetOption.fields());+ if (blob != null) return PathType.DIRECTORY;++ return PathType.FILE;+ }++ @Override+ public String[] listAll(URI path) throws IOException {+ final String blobName = appendTrailingSeparatorIfNecessary(path.toString());++ final String pathStr = blobName;+ final LinkedList<String> result = new LinkedList<>();+ storage+ .list(+ bucketName,+ Storage.BlobListOption.currentDirectory(),+ Storage.BlobListOption.prefix(pathStr),+ Storage.BlobListOption.fields())+ .iterateAll()+ .forEach(+ blob -> {+ assert blob.getName().startsWith(pathStr);+ final String suffixName = blob.getName().substring(pathStr.length());+ if (!suffixName.isEmpty()) {+ // Remove trailing '/' if present+ if (suffixName.endsWith("/")) {+ result.add(suffixName.substring(0, suffixName.length() - 1));+ } else {+ result.add(suffixName);+ }+ }+ });++ return result.toArray(new String[0]);+ }++ @Override+ public IndexInput openInput(URI dirPath, String fileName, IOContext ctx) throws IOException {+ return openInput(dirPath, fileName, ctx, readBufferSizeBytes);+ }++ private IndexInput openInput(URI dirPath, String fileName, IOContext ctx, int bufferSize) {+ String blobName = resolve(dirPath, fileName).toString();++ final BlobId blobId = BlobId.of(bucketName, blobName);+ final Blob blob = storage.get(blobId, Storage.BlobGetOption.fields(Storage.BlobField.SIZE));+ final ReadChannel readChannel = blob.reader();+ readChannel.setChunkSize(bufferSize);++ return new BufferedIndexInput(blobName, bufferSize) {++ @Override+ public long length() {+ return blob.getSize();+ }++ @Override+ protected void readInternal(ByteBuffer b) throws IOException {+ readChannel.read(b);+ }++ @Override+ protected void seekInternal(long pos) throws IOException {+ readChannel.seek(pos);+ }++ @Override+ public void close() throws IOException {+ readChannel.close();+ }+ };+ }++ @Override+ public OutputStream createOutput(URI path) throws IOException {+ final BlobInfo blobInfo = BlobInfo.newBuilder(bucketName, path.toString()).build();+ final WriteChannel writeChannel = storage.writer(blobInfo, getDefaultBlobWriteOptions());++ return Channels.newOutputStream(+ new WritableByteChannel() {+ @Override+ public int write(ByteBuffer src) throws IOException {+ return writeChannel.write(src);+ }++ @Override+ public boolean isOpen() {+ return writeChannel.isOpen();+ }++ @Override+ public void close() throws IOException {+ writeChannel.close();+ }});- }+ }- @Override- public void createDirectory(URI path) throws IOException {- final String name = appendTrailingSeparatorIfNecessary(path.toString());- if (!exists(name)) {- storage.create(BlobInfo.newBuilder(bucketName, name).build());- }+ @Override+ public void createDirectory(URI path) throws IOException {+ final String name = appendTrailingSeparatorIfNecessary(path.toString());+ if (!exists(name)) {+ storage.create(BlobInfo.newBuilder(bucketName, name).build());}-- @Override- public void deleteDirectory(URI path) throws IOException {- List<BlobId> blobIds = allBlobsAtDir(path);- if (!blobIds.isEmpty()) {- storage.delete(blobIds);- } else {- log.debug("Path:{} doesn't have any blobs", path);- }+ }++ @Override+ public void deleteDirectory(URI path) throws IOException {+ List<BlobId> blobIds = allBlobsAtDir(path);+ if (!blobIds.isEmpty()) {+ storage.delete(blobIds);+ } else {+ log.debug("Path:{} doesn't have any blobs", path);}-- protected List<BlobId> allBlobsAtDir(URI path) throws IOException {- final String blobName = appendTrailingSeparatorIfNecessary(path.toString());-- final List<BlobId> result = new ArrayList<>();- final String pathStr = blobName;- storage.list(- bucketName,- Storage.BlobListOption.prefix(pathStr),- Storage.BlobListOption.fields())- .iterateAll().forEach(- blob -> result.add(blob.getBlobId())- );-- return result;-+ }++ protected List<BlobId> allBlobsAtDir(URI path) throws IOException {+ final String blobName = appendTrailingSeparatorIfNecessary(path.toString());++ final List<BlobId> result = new ArrayList<>();+ final String pathStr = blobName;+ storage+ .list(bucketName, Storage.BlobListOption.prefix(pathStr), Storage.BlobListOption.fields())+ .iterateAll()+ .forEach(blob -> result.add(blob.getBlobId()));++ return result;+ }++ @Override+ public void delete(URI path, Collection<String> files, boolean ignoreNoSuchFileException)+ throws IOException {+ if (files.isEmpty()) {+ return;}-- @Override- public void delete(URI path, Collection<String> files, boolean ignoreNoSuchFileException) throws IOException {- if (files.isEmpty()) {- return;- }- final String prefix = appendTrailingSeparatorIfNecessary(path.toString());- List<BlobId> blobDeletes = files.stream()- .map(file -> BlobId.of(bucketName, prefix + file))- .collect(Collectors.toList());- List<Boolean> result = storage.delete(blobDeletes);- if (!ignoreNoSuchFileException) {- int failedDelete = result.indexOf(Boolean.FALSE);- if (failedDelete != -1) {- throw new NoSuchFileException("File " + blobDeletes.get(failedDelete).getName() + " was not found");- }- }+ final String prefix = appendTrailingSeparatorIfNecessary(path.toString());+ List<BlobId> blobDeletes =+ files.stream()+ .map(file -> BlobId.of(bucketName, prefix + file))+ .collect(Collectors.toList());+ List<Boolean> result = storage.delete(blobDeletes);+ if (!ignoreNoSuchFileException) {+ int failedDelete = result.indexOf(Boolean.FALSE);+ if (failedDelete != -1) {+ throw new NoSuchFileException(+ "File " + blobDeletes.get(failedDelete).getName() + " was not found");+ }}-- @Override- public void copyIndexFileFrom(Directory sourceDir, String sourceFileName, URI destDir, String destFileName) throws IOException {- String blobName = destDir.toString();- blobName = appendTrailingSeparatorIfNecessary(blobName);- blobName += destFileName;- final BlobInfo blobInfo = BlobInfo.newBuilder(bucketName, blobName).build();- try (ChecksumIndexInput input = sourceDir.openChecksumInput(sourceFileName, DirectoryFactory.IOCONTEXT_NO_CACHE)) {- if (input.length() <= CodecUtil.footerLength()) {- throw new CorruptIndexException("file is too small:" + input.length(), input);- }- if (input.length() > LARGE_BLOB_THRESHOLD_BYTE_SIZE) {- writeBlobResumable(blobInfo, input);- } else {- writeBlobMultipart(blobInfo, input, (int) input.length());- }- }+ }++ @Override+ public void copyIndexFileFrom(+ Directory sourceDir, String sourceFileName, URI destDir, String destFileName)+ throws IOException {+ String blobName = destDir.toString();+ blobName = appendTrailingSeparatorIfNecessary(blobName);+ blobName += destFileName;+ final BlobInfo blobInfo = BlobInfo.newBuilder(bucketName, blobName).build();+ try (ChecksumIndexInput input =+ sourceDir.openChecksumInput(sourceFileName, DirectoryFactory.IOCONTEXT_NO_CACHE)) {+ if (input.length() <= CodecUtil.footerLength()) {+ throw new CorruptIndexException("file is too small:" + input.length(), input);+ }+ if (input.length() > LARGE_BLOB_THRESHOLD_BYTE_SIZE) {+ writeBlobResumable(blobInfo, input);+ } else {+ writeBlobMultipart(blobInfo, input, (int) input.length());+ }}-- @Override- public void copyIndexFileTo(URI sourceRepo, String sourceFileName, Directory dest, String destFileName) throws IOException {- try {- String blobName = sourceRepo.toString();- blobName = appendTrailingSeparatorIfNecessary(blobName);- blobName += sourceFileName;- final BlobId blobId = BlobId.of(bucketName, blobName);- try (final ReadChannel readChannel = storage.reader(blobId);- IndexOutput output = dest.createOutput(destFileName, DirectoryFactory.IOCONTEXT_NO_CACHE)) {- ByteBuffer buffer = ByteBuffer.allocate(readBufferSizeBytes);- while (readChannel.read(buffer) > 0) {- buffer.flip();- byte[] arr = buffer.array();- output.writeBytes(arr, buffer.position(), buffer.limit() - buffer.position());- buffer.clear();- }- }- } catch (Exception e) {- log.info("Here's an exception e", e);+ }++ @Override+ public void copyIndexFileTo(+ URI sourceRepo, String sourceFileName, Directory dest, String destFileName)+ throws IOException {+ try {+ String blobName = sourceRepo.toString();+ blobName = appendTrailingSeparatorIfNecessary(blobName);+ blobName += sourceFileName;+ final BlobId blobId = BlobId.of(bucketName, blobName);+ try (final ReadChannel readChannel = storage.reader(blobId);+ IndexOutput output =+ dest.createOutput(destFileName, DirectoryFactory.IOCONTEXT_NO_CACHE)) {+ ByteBuffer buffer = ByteBuffer.allocate(readBufferSizeBytes);+ while (readChannel.read(buffer) > 0) {+ buffer.flip();+ byte[] arr = buffer.array();+ output.writeBytes(arr, buffer.position(), buffer.limit() - buffer.position());+ buffer.clear();}+ }+ } catch (Exception e) {+ log.info("Here's an exception e", e);}--- @Override- public void close() throws IOException {-- }-- private void writeBlobMultipart(BlobInfo blobInfo, ChecksumIndexInput indexInput, int blobSize)- throws IOException {- byte[] bytes = new byte[blobSize];- indexInput.readBytes(bytes, 0, blobSize - CodecUtil.footerLength());- long checksum = CodecUtil.checkFooter(indexInput);- ByteBuffer footerBuffer = ByteBuffer.wrap(bytes, blobSize - CodecUtil.footerLength(), CodecUtil.footerLength());- writeFooter(checksum, footerBuffer);- try {- storage.create(blobInfo, bytes, Storage.BlobTargetOption.doesNotExist());- } catch (final StorageException se) {- if (se.getCode() == HTTP_PRECON_FAILED) {- throw new FileAlreadyExistsException(blobInfo.getBlobId().getName(), null, se.getMessage());- }- throw se;- }- }-- private void writeBlobResumable(BlobInfo blobInfo, ChecksumIndexInput indexInput) throws IOException {- try {- final WriteChannel writeChannel = storage.writer(blobInfo, getDefaultBlobWriteOptions());-- ByteBuffer buffer = ByteBuffer.allocate(writeBufferSizeBytes);- writeChannel.setChunkSize(writeBufferSizeBytes);-- long remain = indexInput.length() - CodecUtil.footerLength();- while (remain > 0) {- // reading- int byteReads = (int) Math.min(buffer.capacity(), remain);- indexInput.readBytes(buffer.array(), 0, byteReads);- buffer.position(byteReads);- buffer.flip();-- // writing- writeChannel.write(buffer);- buffer.clear();- remain -= byteReads;- }- long checksum = CodecUtil.checkFooter(indexInput);- ByteBuffer bytes = getFooter(checksum);- writeChannel.write(bytes);- writeChannel.close();- } catch (final StorageException se) {- if (se.getCode() == HTTP_PRECON_FAILED) {- throw new FileAlreadyExistsException(blobInfo.getBlobId().getName(), null, se.getMessage());- }- throw se;- }+ }++ @Override+ public void close() throws IOException {}++ private void writeBlobMultipart(BlobInfo blobInfo, ChecksumIndexInput indexInput, int blobSize)+ throws IOException {+ byte[] bytes = new byte[blobSize];+ indexInput.readBytes(bytes, 0, blobSize - CodecUtil.footerLength());+ long checksum = CodecUtil.checkFooter(indexInput);+ ByteBuffer footerBuffer =+ ByteBuffer.wrap(bytes, blobSize - CodecUtil.footerLength(), CodecUtil.footerLength());+ writeFooter(checksum, footerBuffer);+ try {+ storage.create(blobInfo, bytes, Storage.BlobTargetOption.doesNotExist());+ } catch (final StorageException se) {+ if (se.getCode() == HTTP_PRECON_FAILED) {+ throw new FileAlreadyExistsException(blobInfo.getBlobId().getName(), null, se.getMessage());+ }+ throw se;}+ }++ private void writeBlobResumable(BlobInfo blobInfo, ChecksumIndexInput indexInput)+ throws IOException {+ try {+ final WriteChannel writeChannel = storage.writer(blobInfo, getDefaultBlobWriteOptions());++ ByteBuffer buffer = ByteBuffer.allocate(writeBufferSizeBytes);+ writeChannel.setChunkSize(writeBufferSizeBytes);++ long remain = indexInput.length() - CodecUtil.footerLength();+ while (remain > 0) {+ // reading+ int byteReads = (int) Math.min(buffer.capacity(), remain);+ indexInput.readBytes(buffer.array(), 0, byteReads);+ buffer.position(byteReads);+ buffer.flip();- private ByteBuffer getFooter(long checksum) throws IOException {- ByteBuffer buffer = ByteBuffer.allocate(CodecUtil.footerLength());- writeFooter(checksum, buffer);- return buffer;+ // writing+ writeChannel.write(buffer);+ buffer.clear();+ remain -= byteReads;+ }+ long checksum = CodecUtil.checkFooter(indexInput);+ ByteBuffer bytes = getFooter(checksum);+ writeChannel.write(bytes);+ writeChannel.close();+ } catch (final StorageException se) {+ if (se.getCode() == HTTP_PRECON_FAILED) {+ throw new FileAlreadyExistsException(blobInfo.getBlobId().getName(), null, se.getMessage());+ }+ throw se;}-- private void writeFooter(long checksum, ByteBuffer buffer) throws IOException {- IndexOutput out = new IndexOutput("", "") {-- @Override- public void writeByte(byte b) throws IOException {- buffer.put(b);- }-- @Override- public void writeBytes(byte[] b, int offset, int length) throws IOException {- buffer.put(b, offset, length);- }-- @Override- public void close() throws IOException {-- }-- @Override- public long getFilePointer() {- return 0;- }-- @Override- public long getChecksum() throws IOException {- return checksum;- }+ }++ private ByteBuffer getFooter(long checksum) throws IOException {+ ByteBuffer buffer = ByteBuffer.allocate(CodecUtil.footerLength());+ writeFooter(checksum, buffer);+ return buffer;+ }++ private void writeFooter(long checksum, ByteBuffer buffer) throws IOException {+ IndexOutput out =+ new IndexOutput("", "") {++ @Override+ public void writeByte(byte b) throws IOException {+ buffer.put(b);+ }++ @Override+ public void writeBytes(byte[] b, int offset, int length) throws IOException {+ buffer.put(b, offset, length);+ }++ @Override+ public void close() throws IOException {}++ @Override+ public long getFilePointer() {+ return 0;+ }++ @Override+ public long getChecksum() throws IOException {+ return checksum;+ }};- CodecUtil.writeFooter(out);- buffer.flip();- }+ CodecUtil.writeFooter(out);+ buffer.flip();+ }- protected Storage.BlobWriteOption[] getDefaultBlobWriteOptions() {- return NO_WRITE_OPTIONS;- }+ protected Storage.BlobWriteOption[] getDefaultBlobWriteOptions() {+ return NO_WRITE_OPTIONS;+ }- private String appendTrailingSeparatorIfNecessary(String blobName) {- if (! blobName.endsWith("/")) {- return blobName + "/";- }- return blobName;+ private String appendTrailingSeparatorIfNecessary(String blobName) {+ if (!blobName.endsWith("/")) {+ return blobName + "/";}+ return blobName;+ }}diff --git a/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/GCSConfigParser.java b/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/GCSConfigParser.javaindex ad7f3de95db..9003808eaa1 100644--- a/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/GCSConfigParser.java+++ b/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/GCSConfigParser.java@@ -19,14 +19,11 @@ package org.apache.solr.gcs;import com.google.api.gax.retrying.RetrySettings;import com.google.cloud.storage.StorageOptions;+import java.util.Map;import org.apache.solr.common.util.NamedList;import org.threeten.bp.Duration;-import java.util.Map;--/**- * Parses configuration for {@link GCSBackupRepository} from NamedList and environment variables- */+/** Parses configuration for {@link GCSBackupRepository} from NamedList and environment variables */public class GCSConfigParser {protected static final String GCS_BUCKET_ENV_VAR_NAME = "GCS_BUCKET";protected static final String GCS_CREDENTIAL_ENV_VAR_NAME = "GCS_CREDENTIAL_PATH";@@ -35,15 +32,19 @@ public class GCSConfigParser {private static final String GCS_CREDENTIAL_PARAM_NAME = "gcsCredentialPath";private static final String GCS_WRITE_BUFFER_SIZE_PARAM_NAME = "gcsWriteBufferSizeBytes";private static final String GCS_READ_BUFFER_SIZE_PARAM_NAME = "gcsReadBufferSizeBytes";- private static final String HTTP_CONNECT_TIMEOUT_MILLIS_NAME = "gcsClientHttpConnectTimeoutMillis";+ private static final String HTTP_CONNECT_TIMEOUT_MILLIS_NAME =+ "gcsClientHttpConnectTimeoutMillis";private static final String HTTP_READ_TIMEOUT_MILLIS_NAME = "gcsClientHttpReadTimeoutMillis";private static final String MAX_REQUEST_RETRIES_NAME = "gcsClientMaxRetries";private static final String TOTAL_TIMEOUT_MILLIS_NAME = "gcsClientMaxRequestTimeoutMillis";- private static final String HTTP_INITIAL_RETRY_DELAY_MILLIS_NAME = "gcsClientHttpInitialRetryDelayMillis";- private static final String HTTP_SUBSEQUENT_RETRY_DELAY_MULTIPLIER_NAME = "gcsClientHttpRetryDelayMultiplier";+ private static final String HTTP_INITIAL_RETRY_DELAY_MILLIS_NAME =+ "gcsClientHttpInitialRetryDelayMillis";+ private static final String HTTP_SUBSEQUENT_RETRY_DELAY_MULTIPLIER_NAME =+ "gcsClientHttpRetryDelayMultiplier";private static final String HTTP_MAX_RETRY_DELAY_MILLIS_NAME = "gcsClientHttpMaxRetryDelayMillis";private static final String RPC_INITIAL_TIMEOUT_MILLIS_NAME = "gcsClientRpcInitialTimeoutMillis";- private static final String RPC_SUBSEQUENT_TIMEOUT_MULTIPLIER_NAME = "gcsClientRpcTimeoutMultiplier";+ private static final String RPC_SUBSEQUENT_TIMEOUT_MULTIPLIER_NAME =+ "gcsClientRpcTimeoutMultiplier";private static final String RPC_MAX_TIMEOUT_MILLIS_NAME = "gcsClientRpcMaxTimeoutMillis";private static final String DEFAULT_GCS_BUCKET_VALUE = "solrBackupsBucket";@@ -67,10 +68,19 @@ public class GCSConfigParser {public GCSConfig parseConfiguration(NamedList<?> repoConfig, Map<String, String> envVars) {final String bucketName = parseBucket(repoConfig, envVars);final String credentialPathStr = parseCredentialPath(repoConfig, envVars);- final int writeBufferSizeBytes = getIntOrDefault(repoConfig, GCS_WRITE_BUFFER_SIZE_PARAM_NAME, DEFAULT_GCS_WRITE_BUFFER_SIZE_VALUE);- final int readBufferSizeBytes = getIntOrDefault(repoConfig, GCS_READ_BUFFER_SIZE_PARAM_NAME, DEFAULT_GCS_READ_BUFFER_SIZE_VALUE);+ final int writeBufferSizeBytes =+ getIntOrDefault(+ repoConfig, GCS_WRITE_BUFFER_SIZE_PARAM_NAME, DEFAULT_GCS_WRITE_BUFFER_SIZE_VALUE);+ final int readBufferSizeBytes =+ getIntOrDefault(+ repoConfig, GCS_READ_BUFFER_SIZE_PARAM_NAME, DEFAULT_GCS_READ_BUFFER_SIZE_VALUE);final StorageOptions.Builder storageOptionsBuilder = parseStorageOptions(repoConfig);- return new GCSConfig(bucketName, credentialPathStr, writeBufferSizeBytes, readBufferSizeBytes, storageOptionsBuilder);+ return new GCSConfig(+ bucketName,+ credentialPathStr,+ writeBufferSizeBytes,+ readBufferSizeBytes,+ storageOptionsBuilder);}/*@@ -94,12 +104,15 @@ public class GCSConfigParser {}public static String potentiallyMissingCredentialMsg() {- return "No explicit credential path was provided for this GCSBackupRepository. If Solr is running inside GCP, this " +- "may be expected if GCP's \"Workload Identity\" or a related feature is configured. Solr instances " +- "running outside of GCP however must provide a valid credential path in order to access GCS. These users " +- "should specify their credential path by adding a '" + GCS_CREDENTIAL_PARAM_NAME + "' property to the " +- "repository definition in solconfig, or by setting the path value in an env-var named '" +- GCS_CREDENTIAL_ENV_VAR_NAME + "'.";+ return "No explicit credential path was provided for this GCSBackupRepository. If Solr is running inside GCP, this "+ + "may be expected if GCP's \"Workload Identity\" or a related feature is configured. Solr instances "+ + "running outside of GCP however must provide a valid credential path in order to access GCS. These users "+ + "should specify their credential path by adding a '"+ + GCS_CREDENTIAL_PARAM_NAME+ + "' property to the "+ + "repository definition in solconfig, or by setting the path value in an env-var named '"+ + GCS_CREDENTIAL_ENV_VAR_NAME+ + "'.";}private int getIntOrDefault(NamedList<?> config, String propName, int defaultValue) {@@ -118,27 +131,64 @@ public class GCSConfigParser {private StorageOptions.Builder parseStorageOptions(NamedList<?> repoConfig) {final StorageOptions.Builder builder = StorageOptions.newBuilder();- builder.setTransportOptions(StorageOptions.getDefaultHttpTransportOptions().toBuilder()- .setConnectTimeout(getIntOrDefault(repoConfig, HTTP_CONNECT_TIMEOUT_MILLIS_NAME, DEFAULT_HTTP_CONNECT_TIMEOUT_MILLIS))- .setReadTimeout(getIntOrDefault(repoConfig, HTTP_READ_TIMEOUT_MILLIS_NAME, DEFAULT_HTTP_READ_TIMEOUT_MILLIS))+ builder.setTransportOptions(+ StorageOptions.getDefaultHttpTransportOptions().toBuilder()+ .setConnectTimeout(+ getIntOrDefault(+ repoConfig,+ HTTP_CONNECT_TIMEOUT_MILLIS_NAME,+ DEFAULT_HTTP_CONNECT_TIMEOUT_MILLIS))+ .setReadTimeout(+ getIntOrDefault(+ repoConfig, HTTP_READ_TIMEOUT_MILLIS_NAME, DEFAULT_HTTP_READ_TIMEOUT_MILLIS))+ .build());+ builder.setRetrySettings(+ RetrySettings.newBuilder()+ // All retries+ .setMaxAttempts(+ getIntOrDefault(repoConfig, MAX_REQUEST_RETRIES_NAME, DEFAULT_MAX_RETRIES))+ .setTotalTimeout(+ Duration.ofMillis(+ getIntOrDefault(+ repoConfig, TOTAL_TIMEOUT_MILLIS_NAME, DEFAULT_TOTAL_TIMEOUT_MILLIS)))+ // http requests+ .setInitialRetryDelay(+ Duration.ofMillis(+ getIntOrDefault(+ repoConfig,+ HTTP_INITIAL_RETRY_DELAY_MILLIS_NAME,+ DEFAULT_HTTP_INITIAL_RETRY_DELAY_MILLIS)))+ .setMaxRetryDelay(+ Duration.ofMillis(+ getIntOrDefault(+ repoConfig,+ HTTP_MAX_RETRY_DELAY_MILLIS_NAME,+ DEFAULT_HTTP_MAX_RETRY_DELAY_MILLIS)))+ .setRetryDelayMultiplier(+ getDoubleOrDefault(+ repoConfig,+ HTTP_SUBSEQUENT_RETRY_DELAY_MULTIPLIER_NAME,+ DEFAULT_HTTP_SUBSEQUENT_RETRY_DELAY_MULTIPLIER))+ // rpc requests+ .setInitialRpcTimeout(+ Duration.ofMillis(+ getIntOrDefault(+ repoConfig,+ RPC_INITIAL_TIMEOUT_MILLIS_NAME,+ DEFAULT_RPC_INITIAL_TIMEOUT_MILLIS)))+ .setMaxRpcTimeout(+ Duration.ofMillis(+ getIntOrDefault(+ repoConfig, RPC_MAX_TIMEOUT_MILLIS_NAME, DEFAULT_RPC_MAX_TIMEOUT_MILLIS)))+ .setRpcTimeoutMultiplier(+ getDoubleOrDefault(+ repoConfig,+ RPC_SUBSEQUENT_TIMEOUT_MULTIPLIER_NAME,+ DEFAULT_RPC_SUBSEQUENT_TIMEOUT_MULTIPLIER)).build());- builder.setRetrySettings(RetrySettings.newBuilder()- // All retries- .setMaxAttempts(getIntOrDefault(repoConfig, MAX_REQUEST_RETRIES_NAME, DEFAULT_MAX_RETRIES))- .setTotalTimeout(Duration.ofMillis(getIntOrDefault(repoConfig, TOTAL_TIMEOUT_MILLIS_NAME, DEFAULT_TOTAL_TIMEOUT_MILLIS)))- //http requests- .setInitialRetryDelay(Duration.ofMillis(getIntOrDefault(repoConfig, HTTP_INITIAL_RETRY_DELAY_MILLIS_NAME, DEFAULT_HTTP_INITIAL_RETRY_DELAY_MILLIS)))- .setMaxRetryDelay(Duration.ofMillis(getIntOrDefault(repoConfig, HTTP_MAX_RETRY_DELAY_MILLIS_NAME, DEFAULT_HTTP_MAX_RETRY_DELAY_MILLIS)))- .setRetryDelayMultiplier(getDoubleOrDefault(repoConfig, HTTP_SUBSEQUENT_RETRY_DELAY_MULTIPLIER_NAME, DEFAULT_HTTP_SUBSEQUENT_RETRY_DELAY_MULTIPLIER))- //rpc requests- .setInitialRpcTimeout(Duration.ofMillis(getIntOrDefault(repoConfig, RPC_INITIAL_TIMEOUT_MILLIS_NAME, DEFAULT_RPC_INITIAL_TIMEOUT_MILLIS)))- .setMaxRpcTimeout(Duration.ofMillis(getIntOrDefault(repoConfig, RPC_MAX_TIMEOUT_MILLIS_NAME, DEFAULT_RPC_MAX_TIMEOUT_MILLIS)))- .setRpcTimeoutMultiplier(getDoubleOrDefault(repoConfig, RPC_SUBSEQUENT_TIMEOUT_MULTIPLIER_NAME, DEFAULT_RPC_SUBSEQUENT_TIMEOUT_MULTIPLIER))- .build());return builder;}-public static class GCSConfig {private final StorageOptions.Builder optionsBuilder;private final String bucketName;@@ -146,7 +196,12 @@ public class GCSConfigParser {private final int writeBufferSizeBytes;private final int readBufferSizeBytes;- public GCSConfig(String bucketName, String gcsCredentialPath, int writeBufferSizeBytes, int readBufferSizeBytes, StorageOptions.Builder optionsBuilder) {+ public GCSConfig(+ String bucketName,+ String gcsCredentialPath,+ int writeBufferSizeBytes,+ int readBufferSizeBytes,+ StorageOptions.Builder optionsBuilder) {this.bucketName = bucketName;this.gcsCredentialPath = gcsCredentialPath;this.writeBufferSizeBytes = writeBufferSizeBytes;diff --git a/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/package-info.java b/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/package-info.javaindex e209187846b..c30db9c4a95 100644--- a/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/package-info.java+++ b/solr/modules/gcs-repository/src/java/org/apache/solr/gcs/package-info.java@@ -15,7 +15,5 @@* limitations under the License.*/-/**- * GCS Backup Repository for Solr- */-package org.apache.solr.gcs;\ No newline at end of file+/** GCS Backup Repository for Solr */+package org.apache.solr.gcs;diff --git a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/ConcurrentDelegatingStorage.java b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/ConcurrentDelegatingStorage.javaindex a1074f56f57..d6f5d22e332 100644--- a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/ConcurrentDelegatingStorage.java+++ b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/ConcurrentDelegatingStorage.java@@ -34,7 +34,6 @@ import com.google.cloud.storage.ServiceAccount;import com.google.cloud.storage.Storage;import com.google.cloud.storage.StorageBatch;import com.google.cloud.storage.StorageOptions;-import java.io.IOException;import java.io.InputStream;import java.net.URL;@@ -44,13 +43,15 @@ import java.util.List;import java.util.concurrent.TimeUnit;/**- * Provides a thread-safe wrapper around any {@link Storage} implementations which may not offer thread safety.+ * Provides a thread-safe wrapper around any {@link Storage} implementations which may not offer+ * thread safety.*- * The {@link Storage} implementation provided by google-cloud-nio for in-memory testing is thread-unsafe. This causes- * a problem for backup-restore tests, which have threads reading and writing cloud-store data from the test threads,- * overseer nodes, as well as any nodes hosting individual cores. This class takes a heavyhanded approach to solving- * this by adding synchronization across all instance methods. This allows google-cloud-nio's in-memory fake to be used- * from multiple threads simultaneously as our tests require.+ * <p>The {@link Storage} implementation provided by google-cloud-nio for in-memory testing is+ * thread-unsafe. This causes a problem for backup-restore tests, which have threads reading and+ * writing cloud-store data from the test threads, overseer nodes, as well as any nodes hosting+ * individual cores. This class takes a heavyhanded approach to solving this by adding+ * synchronization across all instance methods. This allows google-cloud-nio's in-memory fake to be+ * used from multiple threads simultaneously as our tests require.*/public class ConcurrentDelegatingStorage implements Storage {@@ -61,210 +62,375 @@ public class ConcurrentDelegatingStorage implements Storage {}@Override- public synchronized Bucket create(BucketInfo bucketInfo, BucketTargetOption... options) { return delegate.create(bucketInfo, options); }+ public synchronized Bucket create(BucketInfo bucketInfo, BucketTargetOption... options) {+ return delegate.create(bucketInfo, options);+ }@Override- public synchronized Blob create(BlobInfo blobInfo, BlobTargetOption... options) { return delegate.create(blobInfo, options); }+ public synchronized Blob create(BlobInfo blobInfo, BlobTargetOption... options) {+ return delegate.create(blobInfo, options);+ }@Override- public synchronized Blob create(BlobInfo blobInfo, byte[] content, BlobTargetOption... options) { return delegate.create(blobInfo, content, options); }+ public synchronized Blob create(BlobInfo blobInfo, byte[] content, BlobTargetOption... options) {+ return delegate.create(blobInfo, content, options);+ }@Override- public synchronized Blob create(BlobInfo blobInfo, byte[] content, int offset, int length, BlobTargetOption... options) { return delegate.create(blobInfo, content, offset, length, options); }+ public synchronized Blob create(+ BlobInfo blobInfo, byte[] content, int offset, int length, BlobTargetOption... options) {+ return delegate.create(blobInfo, content, offset, length, options);+ }@Override- public synchronized Blob create(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) { return delegate.create(blobInfo, content, options); }+ public synchronized Blob create(+ BlobInfo blobInfo, InputStream content, BlobWriteOption... options) {+ return delegate.create(blobInfo, content, options);+ }@Override- public synchronized Blob createFrom(BlobInfo blobInfo, Path path, BlobWriteOption... options) throws IOException { return delegate.createFrom(blobInfo, path, options); }+ public synchronized Blob createFrom(BlobInfo blobInfo, Path path, BlobWriteOption... options)+ throws IOException {+ return delegate.createFrom(blobInfo, path, options);+ }@Override- public synchronized Blob createFrom(BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... options) throws IOException { return delegate.createFrom(blobInfo, path, bufferSize, options); }+ public synchronized Blob createFrom(+ BlobInfo blobInfo, Path path, int bufferSize, BlobWriteOption... options) throws IOException {+ return delegate.createFrom(blobInfo, path, bufferSize, options);+ }@Override- public synchronized Blob createFrom(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) throws IOException { return delegate.createFrom(blobInfo, content, options); }+ public synchronized Blob createFrom(+ BlobInfo blobInfo, InputStream content, BlobWriteOption... options) throws IOException {+ return delegate.createFrom(blobInfo, content, options);+ }@Override- public synchronized Blob createFrom(BlobInfo blobInfo, InputStream content, int bufferSize, BlobWriteOption... options) throws IOException { return delegate.createFrom(blobInfo, content, bufferSize, options); }+ public synchronized Blob createFrom(+ BlobInfo blobInfo, InputStream content, int bufferSize, BlobWriteOption... options)+ throws IOException {+ return delegate.createFrom(blobInfo, content, bufferSize, options);+ }@Override- public synchronized Bucket get(String bucket, BucketGetOption... options) { return delegate.get(bucket, options); }+ public synchronized Bucket get(String bucket, BucketGetOption... options) {+ return delegate.get(bucket, options);+ }@Override- public synchronized Bucket lockRetentionPolicy(BucketInfo bucket, BucketTargetOption... options) { return delegate.lockRetentionPolicy(bucket, options); }+ public synchronized Bucket lockRetentionPolicy(BucketInfo bucket, BucketTargetOption... options) {+ return delegate.lockRetentionPolicy(bucket, options);+ }@Override- public synchronized Blob get(String bucket, String blob, BlobGetOption... options) { return delegate.get(bucket, blob, options); }+ public synchronized Blob get(String bucket, String blob, BlobGetOption... options) {+ return delegate.get(bucket, blob, options);+ }@Override- public synchronized Blob get(BlobId blob, BlobGetOption... options) { return delegate.get(blob, options); }+ public synchronized Blob get(BlobId blob, BlobGetOption... options) {+ return delegate.get(blob, options);+ }@Override- public synchronized Blob get(BlobId blob) { return delegate.get(blob); }+ public synchronized Blob get(BlobId blob) {+ return delegate.get(blob);+ }@Override- public synchronized Page<Bucket> list(BucketListOption... options) { return delegate.list(options); }+ public synchronized Page<Bucket> list(BucketListOption... options) {+ return delegate.list(options);+ }@Override- public synchronized Page<Blob> list(String bucket, BlobListOption... options) { return delegate.list(bucket, options); }+ public synchronized Page<Blob> list(String bucket, BlobListOption... options) {+ return delegate.list(bucket, options);+ }@Override- public synchronized Bucket update(BucketInfo bucketInfo, BucketTargetOption... options) { return delegate.update(bucketInfo, options); }+ public synchronized Bucket update(BucketInfo bucketInfo, BucketTargetOption... options) {+ return delegate.update(bucketInfo, options);+ }@Override- public synchronized Blob update(BlobInfo blobInfo, BlobTargetOption... options) { return delegate.update(blobInfo, options); }+ public synchronized Blob update(BlobInfo blobInfo, BlobTargetOption... options) {+ return delegate.update(blobInfo, options);+ }@Override- public synchronized Blob update(BlobInfo blobInfo) { return delegate.update(blobInfo); }+ public synchronized Blob update(BlobInfo blobInfo) {+ return delegate.update(blobInfo);+ }@Override- public synchronized boolean delete(String bucket, BucketSourceOption... options) { return delegate.delete(bucket, options); }+ public synchronized boolean delete(String bucket, BucketSourceOption... options) {+ return delegate.delete(bucket, options);+ }@Override- public synchronized boolean delete(String bucket, String blob, BlobSourceOption... options) { return delegate.delete(bucket, blob, options); }+ public synchronized boolean delete(String bucket, String blob, BlobSourceOption... options) {+ return delegate.delete(bucket, blob, options);+ }@Override- public synchronized boolean delete(BlobId blob, BlobSourceOption... options) { return delegate.delete(blob, options); }+ public synchronized boolean delete(BlobId blob, BlobSourceOption... options) {+ return delegate.delete(blob, options);+ }@Override- public synchronized boolean delete(BlobId blob) { return delegate.delete(blob); }+ public synchronized boolean delete(BlobId blob) {+ return delegate.delete(blob);+ }@Override- public synchronized Blob compose(ComposeRequest composeRequest) { return delegate.compose(composeRequest); }+ public synchronized Blob compose(ComposeRequest composeRequest) {+ return delegate.compose(composeRequest);+ }@Override- public synchronized CopyWriter copy(CopyRequest copyRequest) { return delegate.copy(copyRequest); }+ public synchronized CopyWriter copy(CopyRequest copyRequest) {+ return delegate.copy(copyRequest);+ }@Override- public synchronized byte[] readAllBytes(String bucket, String blob, BlobSourceOption... options) { return delegate.readAllBytes(bucket, blob, options); }+ public synchronized byte[] readAllBytes(String bucket, String blob, BlobSourceOption... options) {+ return delegate.readAllBytes(bucket, blob, options);+ }@Override- public synchronized byte[] readAllBytes(BlobId blob, BlobSourceOption... options) { return delegate.readAllBytes(blob, options); }+ public synchronized byte[] readAllBytes(BlobId blob, BlobSourceOption... options) {+ return delegate.readAllBytes(blob, options);+ }@Override- public synchronized StorageBatch batch() { return delegate.batch(); }+ public synchronized StorageBatch batch() {+ return delegate.batch();+ }@Override- public synchronized ReadChannel reader(String bucket, String blob, BlobSourceOption... options) { return new ConcurrentReadChannel(this, delegate.reader(bucket, blob, options)); }+ public synchronized ReadChannel reader(String bucket, String blob, BlobSourceOption... options) {+ return new ConcurrentReadChannel(this, delegate.reader(bucket, blob, options));+ }@Override- public synchronized ReadChannel reader(BlobId blob, BlobSourceOption... options) { return new ConcurrentReadChannel(this, delegate.reader(blob, options)); }+ public synchronized ReadChannel reader(BlobId blob, BlobSourceOption... options) {+ return new ConcurrentReadChannel(this, delegate.reader(blob, options));+ }@Override- public synchronized WriteChannel writer(BlobInfo blobInfo, BlobWriteOption... options) { return new ConcurrentWriteChannel(this, delegate.writer(blobInfo, options)); }+ public synchronized WriteChannel writer(BlobInfo blobInfo, BlobWriteOption... options) {+ return new ConcurrentWriteChannel(this, delegate.writer(blobInfo, options));+ }@Override- public synchronized WriteChannel writer(URL signedURL) { return new ConcurrentWriteChannel(this, delegate.writer(signedURL)); }+ public synchronized WriteChannel writer(URL signedURL) {+ return new ConcurrentWriteChannel(this, delegate.writer(signedURL));+ }@Override- public synchronized URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOption... options) { return delegate.signUrl(blobInfo, duration, unit, options); }+ public synchronized URL signUrl(+ BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOption... options) {+ return delegate.signUrl(blobInfo, duration, unit, options);+ }@Override- public synchronized PostPolicyV4 generateSignedPostPolicyV4(BlobInfo blobInfo, long duration, TimeUnit unit, PostPolicyV4.PostFieldsV4 fields, PostPolicyV4.PostConditionsV4 conditions, PostPolicyV4Option... options) { return delegate.generateSignedPostPolicyV4(blobInfo, duration, unit, fields, conditions, options); }+ public synchronized PostPolicyV4 generateSignedPostPolicyV4(+ BlobInfo blobInfo,+ long duration,+ TimeUnit unit,+ PostPolicyV4.PostFieldsV4 fields,+ PostPolicyV4.PostConditionsV4 conditions,+ PostPolicyV4Option... options) {+ return delegate.generateSignedPostPolicyV4(+ blobInfo, duration, unit, fields, conditions, options);+ }@Override- public synchronized PostPolicyV4 generateSignedPostPolicyV4(BlobInfo blobInfo, long duration, TimeUnit unit, PostPolicyV4.PostFieldsV4 fields, PostPolicyV4Option... options) { return delegate.generateSignedPostPolicyV4(blobInfo, duration, unit, fields, options); }+ public synchronized PostPolicyV4 generateSignedPostPolicyV4(+ BlobInfo blobInfo,+ long duration,+ TimeUnit unit,+ PostPolicyV4.PostFieldsV4 fields,+ PostPolicyV4Option... options) {+ return delegate.generateSignedPostPolicyV4(blobInfo, duration, unit, fields, options);+ }@Override- public synchronized PostPolicyV4 generateSignedPostPolicyV4(BlobInfo blobInfo, long duration, TimeUnit unit, PostPolicyV4.PostConditionsV4 conditions, PostPolicyV4Option... options) { return delegate.generateSignedPostPolicyV4(blobInfo, duration, unit, conditions, options); }+ public synchronized PostPolicyV4 generateSignedPostPolicyV4(+ BlobInfo blobInfo,+ long duration,+ TimeUnit unit,+ PostPolicyV4.PostConditionsV4 conditions,+ PostPolicyV4Option... options) {+ return delegate.generateSignedPostPolicyV4(blobInfo, duration, unit, conditions, options);+ }@Override- public synchronized PostPolicyV4 generateSignedPostPolicyV4(BlobInfo blobInfo, long duration, TimeUnit unit, PostPolicyV4Option... options) { return delegate.generateSignedPostPolicyV4(blobInfo, duration, unit, options); }+ public synchronized PostPolicyV4 generateSignedPostPolicyV4(+ BlobInfo blobInfo, long duration, TimeUnit unit, PostPolicyV4Option... options) {+ return delegate.generateSignedPostPolicyV4(blobInfo, duration, unit, options);+ }@Override- public synchronized List<Blob> get(BlobId... blobIds) { return delegate.get(blobIds); }+ public synchronized List<Blob> get(BlobId... blobIds) {+ return delegate.get(blobIds);+ }@Override- public synchronized List<Blob> get(Iterable<BlobId> blobIds) { return delegate.get(blobIds); }+ public synchronized List<Blob> get(Iterable<BlobId> blobIds) {+ return delegate.get(blobIds);+ }@Override- public synchronized List<Blob> update(BlobInfo... blobInfos) { return delegate.update(blobInfos); }+ public synchronized List<Blob> update(BlobInfo... blobInfos) {+ return delegate.update(blobInfos);+ }@Override- public synchronized List<Blob> update(Iterable<BlobInfo> blobInfos) { return delegate.update(blobInfos); }+ public synchronized List<Blob> update(Iterable<BlobInfo> blobInfos) {+ return delegate.update(blobInfos);+ }@Override- public synchronized List<Boolean> delete(BlobId... blobIds) { return delegate.delete(blobIds); }+ public synchronized List<Boolean> delete(BlobId... blobIds) {+ return delegate.delete(blobIds);+ }@Override- public synchronized List<Boolean> delete(Iterable<BlobId> blobIds) { return delegate.delete(blobIds); }+ public synchronized List<Boolean> delete(Iterable<BlobId> blobIds) {+ return delegate.delete(blobIds);+ }@Override- public synchronized Acl getAcl(String bucket, Acl.Entity entity, BucketSourceOption... options) { return delegate.getAcl(bucket, entity, options); }+ public synchronized Acl getAcl(String bucket, Acl.Entity entity, BucketSourceOption... options) {+ return delegate.getAcl(bucket, entity, options);+ }@Override- public synchronized Acl getAcl(String bucket, Acl.Entity entity) { return delegate.getAcl(bucket, entity); }+ public synchronized Acl getAcl(String bucket, Acl.Entity entity) {+ return delegate.getAcl(bucket, entity);+ }@Override- public synchronized boolean deleteAcl(String bucket, Acl.Entity entity, BucketSourceOption... options) { return delegate.deleteAcl(bucket, entity, options); }+ public synchronized boolean deleteAcl(+ String bucket, Acl.Entity entity, BucketSourceOption... options) {+ return delegate.deleteAcl(bucket, entity, options);+ }@Override- public synchronized boolean deleteAcl(String bucket, Acl.Entity entity) { return delegate.deleteAcl(bucket, entity); }+ public synchronized boolean deleteAcl(String bucket, Acl.Entity entity) {+ return delegate.deleteAcl(bucket, entity);+ }@Override- public synchronized Acl createAcl(String bucket, Acl acl, BucketSourceOption... options) { return delegate.createAcl(bucket, acl, options); }+ public synchronized Acl createAcl(String bucket, Acl acl, BucketSourceOption... options) {+ return delegate.createAcl(bucket, acl, options);+ }@Override- public synchronized Acl createAcl(String bucket, Acl acl) { return delegate.createAcl(bucket, acl); }+ public synchronized Acl createAcl(String bucket, Acl acl) {+ return delegate.createAcl(bucket, acl);+ }@Override- public synchronized Acl updateAcl(String bucket, Acl acl, BucketSourceOption... options) { return delegate.updateAcl(bucket, acl, options); }+ public synchronized Acl updateAcl(String bucket, Acl acl, BucketSourceOption... options) {+ return delegate.updateAcl(bucket, acl, options);+ }@Override- public synchronized Acl updateAcl(String bucket, Acl acl) { return delegate.updateAcl(bucket, acl); }+ public synchronized Acl updateAcl(String bucket, Acl acl) {+ return delegate.updateAcl(bucket, acl);+ }@Override- public synchronized List<Acl> listAcls(String bucket, BucketSourceOption... options) { return delegate.listAcls(bucket, options); }+ public synchronized List<Acl> listAcls(String bucket, BucketSourceOption... options) {+ return delegate.listAcls(bucket, options);+ }@Override- public synchronized List<Acl> listAcls(String bucket) { return delegate.listAcls(bucket); }+ public synchronized List<Acl> listAcls(String bucket) {+ return delegate.listAcls(bucket);+ }@Override- public synchronized Acl getDefaultAcl(String bucket, Acl.Entity entity) { return delegate.getDefaultAcl(bucket, entity); }+ public synchronized Acl getDefaultAcl(String bucket, Acl.Entity entity) {+ return delegate.getDefaultAcl(bucket, entity);+ }@Override- public synchronized boolean deleteDefaultAcl(String bucket, Acl.Entity entity) { return delegate.deleteDefaultAcl(bucket, entity); }+ public synchronized boolean deleteDefaultAcl(String bucket, Acl.Entity entity) {+ return delegate.deleteDefaultAcl(bucket, entity);+ }@Override- public synchronized Acl createDefaultAcl(String bucket, Acl acl) { return delegate.createDefaultAcl(bucket, acl); }+ public synchronized Acl createDefaultAcl(String bucket, Acl acl) {+ return delegate.createDefaultAcl(bucket, acl);+ }@Override- public synchronized Acl updateDefaultAcl(String bucket, Acl acl) { return delegate.updateDefaultAcl(bucket, acl); }+ public synchronized Acl updateDefaultAcl(String bucket, Acl acl) {+ return delegate.updateDefaultAcl(bucket, acl);+ }@Override- public synchronized List<Acl> listDefaultAcls(String bucket) { return delegate.listDefaultAcls(bucket); }+ public synchronized List<Acl> listDefaultAcls(String bucket) {+ return delegate.listDefaultAcls(bucket);+ }@Override- public synchronized Acl getAcl(BlobId blob, Acl.Entity entity) { return delegate.getAcl(blob, entity); }+ public synchronized Acl getAcl(BlobId blob, Acl.Entity entity) {+ return delegate.getAcl(blob, entity);+ }@Override- public synchronized boolean deleteAcl(BlobId blob, Acl.Entity entity) { return delegate.deleteAcl(blob, entity); }+ public synchronized boolean deleteAcl(BlobId blob, Acl.Entity entity) {+ return delegate.deleteAcl(blob, entity);+ }@Override- public synchronized Acl createAcl(BlobId blob, Acl acl) { return delegate.createAcl(blob, acl); }+ public synchronized Acl createAcl(BlobId blob, Acl acl) {+ return delegate.createAcl(blob, acl);+ }@Override- public synchronized Acl updateAcl(BlobId blob, Acl acl) { return delegate.updateAcl(blob, acl); }+ public synchronized Acl updateAcl(BlobId blob, Acl acl) {+ return delegate.updateAcl(blob, acl);+ }@Override- public synchronized List<Acl> listAcls(BlobId blob) { return delegate.listAcls(blob); }+ public synchronized List<Acl> listAcls(BlobId blob) {+ return delegate.listAcls(blob);+ }@Override- public synchronized HmacKey createHmacKey(ServiceAccount serviceAccount, CreateHmacKeyOption... options) { return delegate.createHmacKey(serviceAccount, options); }+ public synchronized HmacKey createHmacKey(+ ServiceAccount serviceAccount, CreateHmacKeyOption... options) {+ return delegate.createHmacKey(serviceAccount, options);+ }@Override- public synchronized Page<HmacKey.HmacKeyMetadata> listHmacKeys(ListHmacKeysOption... options) { return delegate.listHmacKeys(options); }+ public synchronized Page<HmacKey.HmacKeyMetadata> listHmacKeys(ListHmacKeysOption... options) {+ return delegate.listHmacKeys(options);+ }@Override- public synchronized HmacKey.HmacKeyMetadata getHmacKey(String accessId, GetHmacKeyOption... options) { return delegate.getHmacKey(accessId, options); }+ public synchronized HmacKey.HmacKeyMetadata getHmacKey(+ String accessId, GetHmacKeyOption... options) {+ return delegate.getHmacKey(accessId, options);+ }@Override- public synchronized void deleteHmacKey(HmacKey.HmacKeyMetadata hmacKeyMetadata, DeleteHmacKeyOption... options) {+ public synchronized void deleteHmacKey(+ HmacKey.HmacKeyMetadata hmacKeyMetadata, DeleteHmacKeyOption... options) {delegate.deleteHmacKey(hmacKeyMetadata, options);}@Override- public synchronized HmacKey.HmacKeyMetadata updateHmacKeyState(HmacKey.HmacKeyMetadata hmacKeyMetadata, HmacKey.HmacKeyState state, UpdateHmacKeyOption... options) {+ public synchronized HmacKey.HmacKeyMetadata updateHmacKeyState(+ HmacKey.HmacKeyMetadata hmacKeyMetadata,+ HmacKey.HmacKeyState state,+ UpdateHmacKeyOption... options) {return delegate.updateHmacKeyState(hmacKeyMetadata, state, options);}@@ -274,12 +440,14 @@ public class ConcurrentDelegatingStorage implements Storage {}@Override- public synchronized Policy setIamPolicy(String bucket, Policy policy, BucketSourceOption... options) {+ public synchronized Policy setIamPolicy(+ String bucket, Policy policy, BucketSourceOption... options) {return delegate.setIamPolicy(bucket, policy, options);}@Override- public synchronized List<Boolean> testIamPermissions(String bucket, List<String> permissions, BucketSourceOption... options) {+ public synchronized List<Boolean> testIamPermissions(+ String bucket, List<String> permissions, BucketSourceOption... options) {return delegate.testIamPermissions(bucket, permissions, options);}diff --git a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSBackupRepositoryTest.java b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSBackupRepositoryTest.javaindex 4676ef7456a..03aa0aff549 100644--- a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSBackupRepositoryTest.java+++ b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSBackupRepositoryTest.java@@ -17,57 +17,53 @@package org.apache.solr.gcs;-import org.apache.solr.cloud.api.collections.AbstractBackupRepositoryTest;-import org.apache.solr.common.util.NamedList;-import org.apache.solr.core.backup.repository.BackupRepository;-import org.junit.AfterClass;-import org.junit.Test;+import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION;+import static org.apache.solr.gcs.GCSConfigParser.GCS_BUCKET_ENV_VAR_NAME;+import static org.apache.solr.gcs.GCSConfigParser.GCS_CREDENTIAL_ENV_VAR_NAME;import java.net.URI;import java.net.URISyntaxException;import java.util.HashMap;import java.util.Map;+import org.apache.solr.cloud.api.collections.AbstractBackupRepositoryTest;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.core.backup.repository.BackupRepository;+import org.junit.AfterClass;+import org.junit.Test;-import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION;-import static org.apache.solr.gcs.GCSConfigParser.GCS_BUCKET_ENV_VAR_NAME;-import static org.apache.solr.gcs.GCSConfigParser.GCS_CREDENTIAL_ENV_VAR_NAME;--/**- * Unit tests for {@link GCSBackupRepository} that use an in-memory Storage object- */+/** Unit tests for {@link GCSBackupRepository} that use an in-memory Storage object */public class GCSBackupRepositoryTest extends AbstractBackupRepositoryTest {- @AfterClass- public static void tearDownClass() throws Exception {- LocalStorageGCSBackupRepository.clearStashedStorage();- }+ @AfterClass+ public static void tearDownClass() throws Exception {+ LocalStorageGCSBackupRepository.clearStashedStorage();+ }- @Override- protected BackupRepository getRepository() {- final NamedList<Object> config = new NamedList<>();- config.add(BACKUP_LOCATION, "backup1");- final GCSBackupRepository repository = new LocalStorageGCSBackupRepository();- repository.init(config);+ @Override+ protected BackupRepository getRepository() {+ final NamedList<Object> config = new NamedList<>();+ config.add(BACKUP_LOCATION, "backup1");+ final GCSBackupRepository repository = new LocalStorageGCSBackupRepository();+ repository.init(config);- return repository;- }+ return repository;+ }- @Override- protected URI getBaseUri() throws URISyntaxException {- return new URI("tmp");- }+ @Override+ protected URI getBaseUri() throws URISyntaxException {+ return new URI("tmp");+ }- @Test- public void testInitStoreDoesNotFailWithMissingCredentials()- {- Map<String, String> config = new HashMap<>();- config.put(GCS_BUCKET_ENV_VAR_NAME, "a_bucket_name");- // explicitly setting credential name to null; will work inside google-cloud project- config.put(GCS_CREDENTIAL_ENV_VAR_NAME, null);- config.put(BACKUP_LOCATION, "/==");+ @Test+ public void testInitStoreDoesNotFailWithMissingCredentials() {+ Map<String, String> config = new HashMap<>();+ config.put(GCS_BUCKET_ENV_VAR_NAME, "a_bucket_name");+ // explicitly setting credential name to null; will work inside google-cloud project+ config.put(GCS_CREDENTIAL_ENV_VAR_NAME, null);+ config.put(BACKUP_LOCATION, "/==");- BackupRepository gcsBackupRepository = getRepository();+ BackupRepository gcsBackupRepository = getRepository();- gcsBackupRepository.init(new NamedList<>(config));- }+ gcsBackupRepository.init(new NamedList<>(config));+ }}diff --git a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.java b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.javaindex b567252ae63..d72c0d4801a 100644--- a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.java+++ b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.java@@ -18,6 +18,7 @@package org.apache.solr.gcs;import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;+import java.lang.invoke.MethodHandles;import org.apache.lucene.util.LuceneTestCase;import org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest;import org.junit.AfterClass;@@ -25,71 +26,71 @@ import org.junit.BeforeClass;import org.slf4j.Logger;import org.slf4j.LoggerFactory;-import java.lang.invoke.MethodHandles;-@ThreadLeakLingering(linger = 10)[email protected]({"SimpleText"}) // Backups do checksum validation against a footer value not present in 'SimpleText'+ "SimpleText"+}) // Backups do checksum validation against a footer value not present in 'SimpleText'public class GCSIncrementalBackupTest extends AbstractIncrementalBackupTest {- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());- public static final String SOLR_XML = "<solr>\n" +- "\n" +- " <str name=\"shareSchema\">${shareSchema:false}</str>\n" +- " <str name=\"configSetBaseDir\">${configSetBaseDir:configsets}</str>\n" +- " <str name=\"coreRootDirectory\">${coreRootDirectory:.}</str>\n" +- "\n" +- " <shardHandlerFactory name=\"shardHandlerFactory\" class=\"HttpShardHandlerFactory\">\n" +- " <str name=\"urlScheme\">${urlScheme:}</str>\n" +- " <int name=\"socketTimeout\">${socketTimeout:90000}</int>\n" +- " <int name=\"connTimeout\">${connTimeout:15000}</int>\n" +- " </shardHandlerFactory>\n" +- "\n" +- " <solrcloud>\n" +- " <str name=\"host\">127.0.0.1</str>\n" +- " <int name=\"hostPort\">${hostPort:8983}</int>\n" +- " <str name=\"hostContext\">${hostContext:solr}</str>\n" +- " <int name=\"zkClientTimeout\">${solr.zkclienttimeout:30000}</int>\n" +- " <bool name=\"genericCoreNodeNames\">${genericCoreNodeNames:true}</bool>\n" +- " <int name=\"leaderVoteWait\">10000</int>\n" +- " <int name=\"distribUpdateConnTimeout\">${distribUpdateConnTimeout:45000}</int>\n" +- " <int name=\"distribUpdateSoTimeout\">${distribUpdateSoTimeout:340000}</int>\n" +- " </solrcloud>\n" +- " \n" +- " <backup>\n" +- " <repository name=\"trackingBackupRepository\" class=\"org.apache.solr.core.TrackingBackupRepository\"> \n" +- " <str name=\"delegateRepoName\">localfs</str>\n" +- " </repository>\n" +- " <repository name=\"localfs\" class=\"org.apache.solr.gcs.LocalStorageGCSBackupRepository\"> \n" +- " <str name=\"gcsBucket\">someBucketName</str>\n" +- " <str name=\"location\">backup1</str>\n" +- " </repository>\n" +- " </backup>\n" +- " \n" +- "</solr>\n";-- private static String backupLocation;+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());+ public static final String SOLR_XML =+ "<solr>\n"+ + "\n"+ + " <str name=\"shareSchema\">${shareSchema:false}</str>\n"+ + " <str name=\"configSetBaseDir\">${configSetBaseDir:configsets}</str>\n"+ + " <str name=\"coreRootDirectory\">${coreRootDirectory:.}</str>\n"+ + "\n"+ + " <shardHandlerFactory name=\"shardHandlerFactory\" class=\"HttpShardHandlerFactory\">\n"+ + " <str name=\"urlScheme\">${urlScheme:}</str>\n"+ + " <int name=\"socketTimeout\">${socketTimeout:90000}</int>\n"+ + " <int name=\"connTimeout\">${connTimeout:15000}</int>\n"+ + " </shardHandlerFactory>\n"+ + "\n"+ + " <solrcloud>\n"+ + " <str name=\"host\">127.0.0.1</str>\n"+ + " <int name=\"hostPort\">${hostPort:8983}</int>\n"+ + " <str name=\"hostContext\">${hostContext:solr}</str>\n"+ + " <int name=\"zkClientTimeout\">${solr.zkclienttimeout:30000}</int>\n"+ + " <bool name=\"genericCoreNodeNames\">${genericCoreNodeNames:true}</bool>\n"+ + " <int name=\"leaderVoteWait\">10000</int>\n"+ + " <int name=\"distribUpdateConnTimeout\">${distribUpdateConnTimeout:45000}</int>\n"+ + " <int name=\"distribUpdateSoTimeout\">${distribUpdateSoTimeout:340000}</int>\n"+ + " </solrcloud>\n"+ + " \n"+ + " <backup>\n"+ + " <repository name=\"trackingBackupRepository\" class=\"org.apache.solr.core.TrackingBackupRepository\"> \n"+ + " <str name=\"delegateRepoName\">localfs</str>\n"+ + " </repository>\n"+ + " <repository name=\"localfs\" class=\"org.apache.solr.gcs.LocalStorageGCSBackupRepository\"> \n"+ + " <str name=\"gcsBucket\">someBucketName</str>\n"+ + " <str name=\"location\">backup1</str>\n"+ + " </repository>\n"+ + " </backup>\n"+ + " \n"+ + "</solr>\n";- @BeforeClass- public static void setupClass() throws Exception {+ private static String backupLocation;- configureCluster(NUM_SHARDS)// nodes- .addConfig("conf1", getFile("conf/solrconfig.xml").getParentFile().toPath())- .withSolrXml(SOLR_XML)- .configure();- }+ @BeforeClass+ public static void setupClass() throws Exception {- @AfterClass- public static void tearDownClass() throws Exception {- LocalStorageGCSBackupRepository.clearStashedStorage();- }+ configureCluster(NUM_SHARDS) // nodes+ .addConfig("conf1", getFile("conf/solrconfig.xml").getParentFile().toPath())+ .withSolrXml(SOLR_XML)+ .configure();+ }- @Override- public String getCollectionNamePrefix() {- return "backuprestore";- }+ @AfterClass+ public static void tearDownClass() throws Exception {+ LocalStorageGCSBackupRepository.clearStashedStorage();+ }- @Override- public String getBackupLocation() {- return "backup1";- }+ @Override+ public String getCollectionNamePrefix() {+ return "backuprestore";+ }+ @Override+ public String getBackupLocation() {+ return "backup1";+ }}diff --git a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/LocalStorageGCSBackupRepository.java b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/LocalStorageGCSBackupRepository.javaindex 30d0df137f8..84460e53e35 100644--- a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/LocalStorageGCSBackupRepository.java+++ b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/LocalStorageGCSBackupRepository.java@@ -17,12 +17,13 @@package org.apache.solr.gcs;+import static org.apache.lucene.util.LuceneTestCase.assumeFalse;+import com.google.cloud.storage.BlobId;import com.google.cloud.storage.Storage;import com.google.cloud.storage.StorageException;import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;import com.google.common.collect.Lists;-import java.io.IOException;import java.net.URI;import java.nio.file.NoSuchFileException;@@ -30,16 +31,16 @@ import java.util.Collection;import java.util.List;import java.util.stream.Collectors;-import static org.apache.lucene.util.LuceneTestCase.assumeFalse;-public class LocalStorageGCSBackupRepository extends GCSBackupRepository {protected static Storage stashedStorage = null;@Overridepublic Storage initStorage() {- // LocalStorageHelper produces 'Storage' objects that track blob store state in non-static memory unique to each- // Storage instance. For various components in Solr to have a coherent view of the blob-space then, they need to+ // LocalStorageHelper produces 'Storage' objects that track blob store state in non-static+ // memory unique to each+ // Storage instance. For various components in Solr to have a coherent view of the blob-space+ // then, they need to// share a single 'Storage' object.synchronized (LocalStorageGCSBackupRepository.class) {storage = getSingletonStorage();@@ -54,10 +55,12 @@ public class LocalStorageGCSBackupRepository extends GCSBackupRepository {}}- // A reimplementation of delete functionality that avoids batching. Batching is ideal in production use cases, but+ // A reimplementation of delete functionality that avoids batching. Batching is ideal in+ // production use cases, but// isn't supported by the in-memory Storage implementation provided by LocalStorageHelper@Override- public void delete(URI path, Collection<String> files, boolean ignoreNoSuchFileException) throws IOException {+ public void delete(URI path, Collection<String> files, boolean ignoreNoSuchFileException)+ throws IOException {final List<Boolean> results = Lists.newArrayList();final List<String> filesOrdered = files.stream().collect(Collectors.toList());@@ -89,9 +92,12 @@ public class LocalStorageGCSBackupRepository extends GCSBackupRepository {return stashedStorage;}- // FakeStorageRpc isn't thread-safe, which causes flaky test failures when multiple cores attempt to backup files- // simultaneously. We work around this here by wrapping it in a delegating instance that adds a measure of thread safety.- stashedStorage = new ConcurrentDelegatingStorage(LocalStorageHelper.customOptions(false).getService());+ // FakeStorageRpc isn't thread-safe, which causes flaky test failures when multiple cores+ // attempt to backup files+ // simultaneously. We work around this here by wrapping it in a delegating instance that adds a+ // measure of thread safety.+ stashedStorage =+ new ConcurrentDelegatingStorage(LocalStorageHelper.customOptions(false).getService());return stashedStorage;}@@ -103,10 +109,11 @@ public class LocalStorageGCSBackupRepository extends GCSBackupRepository {} catch (Exception e) {final Throwable cause = e.getCause();if (cause != null) {- assumeFalse("This test uses a GCS mock library that is incompatible with the current default locale",- e instanceof StorageException &&- cause.getMessage().contains("Invalid date/time format") &&- cause instanceof NumberFormatException);+ assumeFalse(+ "This test uses a GCS mock library that is incompatible with the current default locale",+ e instanceof StorageException+ && cause.getMessage().contains("Invalid date/time format")+ && cause instanceof NumberFormatException);}}}