Apache Solr open-source search software
SOLR-17302: Convert /cluster filestore APIs to JAX-RS (reinstating after revert) (#2507)
Jason Gerlowski26365679bde2
diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.javanew file mode 100644index 00000000000..07746b5e92d--- /dev/null+++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java@@ -0,0 +1,68 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.client.api.endpoint;++import static org.apache.solr.client.api.util.Constants.GENERIC_ENTITY_PROPERTY;++import io.swagger.v3.oas.annotations.Operation;+import io.swagger.v3.oas.annotations.Parameter;+import io.swagger.v3.oas.annotations.extensions.Extension;+import io.swagger.v3.oas.annotations.extensions.ExtensionProperty;+import io.swagger.v3.oas.annotations.parameters.RequestBody;+import jakarta.ws.rs.DELETE;+import jakarta.ws.rs.PUT;+import jakarta.ws.rs.Path;+import jakarta.ws.rs.PathParam;+import jakarta.ws.rs.QueryParam;+import java.io.InputStream;+import java.util.List;+import org.apache.solr.client.api.model.SolrJerseyResponse;+import org.apache.solr.client.api.model.UploadToFileStoreResponse;++@Path("/cluster")+public interface ClusterFileStoreApis {+ // TODO Better understand the purpose of the 'sig' parameter and improve docs here.+ @PUT+ @Operation(+ summary = "Upload a file to the filestore.",+ tags = {"file-store"})+ @Path("/files{filePath:.+}")+ UploadToFileStoreResponse uploadFile(+ @Parameter(description = "File store path") @PathParam("filePath") String filePath,+ @Parameter(description = "Signature(s) for the file being uploaded") @QueryParam("sig")+ List<String> sig,+ @Parameter(description = "File content to be stored in the filestore")+ @RequestBody(+ required = true,+ extensions = {+ @Extension(+ properties = {+ @ExtensionProperty(name = GENERIC_ENTITY_PROPERTY, value = "true")+ })+ })+ InputStream requestBody);++ @DELETE+ @Operation(+ summary = "Delete a file or directory from the filestore.",+ tags = {"file-store"})+ @Path("/files{path:.+}")+ SolrJerseyResponse deleteFile(+ @Parameter(description = "Path to a file or directory within the filestore")+ @PathParam("path")+ String path);+}diff --git a/solr/api/src/java/org/apache/solr/client/api/model/UploadToFileStoreResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/UploadToFileStoreResponse.javanew file mode 100644index 00000000000..4492dce7daa--- /dev/null+++ b/solr/api/src/java/org/apache/solr/client/api/model/UploadToFileStoreResponse.java@@ -0,0 +1,25 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.client.api.model;++import com.fasterxml.jackson.annotation.JsonProperty;++public class UploadToFileStoreResponse extends SolrJerseyResponse {++ @JsonProperty public String file;+ @JsonProperty public String message;+}diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.javaindex f21bbc592e4..a05bfed4e7d 100644--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java@@ -104,6 +104,9 @@ import org.apache.solr.common.util.Utils;import org.apache.solr.core.DirectoryFactory.DirContext;import org.apache.solr.core.backup.repository.BackupRepository;import org.apache.solr.core.backup.repository.BackupRepositoryFactory;+import org.apache.solr.filestore.ClusterFileStore;+import org.apache.solr.filestore.DistribFileStore;+import org.apache.solr.filestore.FileStore;import org.apache.solr.filestore.FileStoreAPI;import org.apache.solr.handler.ClusterAPI;import org.apache.solr.handler.RequestHandlerBase;@@ -294,7 +297,9 @@ public class CoreContainer {private volatile ClusterEventProducer clusterEventProducer;private DelegatingPlacementPluginFactory placementPluginFactory;+ private DistribFileStore fileStore;private FileStoreAPI fileStoreAPI;+ private ClusterFileStore clusterFileStoreAPI;private SolrPackageLoader packageLoader;private final Set<Path> allowPaths;@@ -723,8 +728,8 @@ public class CoreContainer {return packageLoader;}- public FileStoreAPI getFileStoreAPI() {- return fileStoreAPI;+ public FileStore getFileStore() {+ return fileStore;}public SolrCache<?, ?> getCache(String name) {@@ -856,9 +861,11 @@ public class CoreContainer {(PublicKeyHandler) containerHandlers.get(PublicKeyHandler.PATH));pkiAuthenticationSecurityBuilder.initializeMetrics(solrMetricsContext, "/authentication/pki");+ fileStore = new DistribFileStore(this);fileStoreAPI = new FileStoreAPI(this);registerV2ApiIfEnabled(fileStoreAPI.readAPI);registerV2ApiIfEnabled(fileStoreAPI.writeAPI);+ registerV2ApiIfEnabled(ClusterFileStore.class);packageLoader = new SolrPackageLoader(this);registerV2ApiIfEnabled(packageLoader.getPackageAPI().editAPI);@@ -1150,6 +1157,15 @@ public class CoreContainer {.in(Singleton.class);}})+ .register(+ new AbstractBinder() {+ @Override+ protected void configure() {+ bindFactory(new InjectionFactories.SingletonFactory<>(fileStore))+ .to(DistribFileStore.class)+ .in(Singleton.class);+ }+ }).register(new AbstractBinder() {@Overridediff --git a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.javanew file mode 100644index 00000000000..a664e5699cb--- /dev/null+++ b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java@@ -0,0 +1,257 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.filestore;++import static java.nio.charset.StandardCharsets.UTF_8;++import jakarta.inject.Inject;+import java.io.IOException;+import java.io.InputStream;+import java.lang.invoke.MethodHandles;+import java.nio.ByteBuffer;+import java.util.HashMap;+import java.util.List;+import java.util.Map;+import org.apache.commons.codec.digest.DigestUtils;+import org.apache.solr.api.JerseyResource;+import org.apache.solr.client.api.endpoint.ClusterFileStoreApis;+import org.apache.solr.client.api.model.SolrJerseyResponse;+import org.apache.solr.client.api.model.UploadToFileStoreResponse;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.util.StrUtils;+import org.apache.solr.core.CoreContainer;+import org.apache.solr.jersey.PermissionName;+import org.apache.solr.pkg.PackageAPI;+import org.apache.solr.request.SolrQueryRequest;+import org.apache.solr.response.SolrQueryResponse;+import org.apache.solr.security.PermissionNameProvider;+import org.apache.solr.util.CryptoKeys;+import org.apache.zookeeper.CreateMode;+import org.apache.zookeeper.KeeperException;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++public class ClusterFileStore extends JerseyResource implements ClusterFileStoreApis {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());+ public static final String FILESTORE_DIRECTORY = "filestore";+ public static final String TRUSTED_DIR = "_trusted_";+ public static final String KEYS_DIR = "/_trusted_/keys";+ static final String TMP_ZK_NODE = "/fileStoreWriteInProgress";++ private final CoreContainer coreContainer;+ private final SolrQueryRequest req;+ private final SolrQueryResponse rsp;+ private final FileStore fileStore;++ @Inject+ public ClusterFileStore(+ CoreContainer coreContainer,+ DistribFileStore fileStore,+ SolrQueryRequest req,+ SolrQueryResponse rsp) {+ this.coreContainer = coreContainer;+ this.req = req;+ this.rsp = rsp;+ this.fileStore = fileStore;+ }++ @Override+ @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM)+ public UploadToFileStoreResponse uploadFile(+ String filePath, List<String> sig, InputStream requestBody) {+ final var response = instantiateJerseyResponse(UploadToFileStoreResponse.class);+ if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) {+ throw new RuntimeException(PackageAPI.ERR_MSG);+ }+ try {+ coreContainer+ .getZkController()+ .getZkClient()+ .create(TMP_ZK_NODE, "true".getBytes(UTF_8), CreateMode.EPHEMERAL, true);++ if (requestBody == null)+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "no payload");+ if (filePath == null) {+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No path");+ }+ validateName(filePath, true);+ try {+ byte[] buf = requestBody.readAllBytes();+ List<String> signatures = readSignatures(sig, buf);+ FileStoreAPI.MetaData meta = _createJsonMetaData(buf, signatures);+ FileStore.FileType type = fileStore.getType(filePath, true);+ if (type == FileStore.FileType.FILE) {+ // a file already exist at the same path+ fileStore.get(+ filePath,+ fileEntry -> {+ if (meta.equals(fileEntry.meta)) {+ // the file content is same too. this is an idempotent put+ // do not throw an error+ response.file = filePath;+ response.message = "File with same metadata exists ";+ }+ },+ true);+ // 'message' only set in the "already exists w/ same content" case, so we're done!+ if (response.message != null) {+ return response;+ }+ } else if (type != FileStore.FileType.NOFILE) {+ throw new SolrException(+ SolrException.ErrorCode.BAD_REQUEST, "Path already exists " + filePath);+ }++ fileStore.put(new FileStore.FileEntry(ByteBuffer.wrap(buf), meta, filePath));+ response.file = filePath;+ } catch (IOException e) {+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);+ }+ } catch (InterruptedException e) {+ log.error("Unexpected error", e);+ } catch (KeeperException.NodeExistsException e) {+ throw new SolrException(+ SolrException.ErrorCode.SERVER_ERROR, "A write is already in process , try later");+ } catch (KeeperException e) {+ log.error("Unexpected error", e);+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e.getMessage());+ } finally {+ try {+ coreContainer.getZkController().getZkClient().delete(TMP_ZK_NODE, -1, true);+ } catch (Exception e) {+ log.error("Unexpected error ", e);+ }+ }++ return response;+ }++ @Override+ @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM)+ public SolrJerseyResponse deleteFile(String filePath) {+ final var response = instantiateJerseyResponse(SolrJerseyResponse.class);+ if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) {+ throw new RuntimeException(PackageAPI.ERR_MSG);+ }++ try {+ coreContainer+ .getZkController()+ .getZkClient()+ .create(TMP_ZK_NODE, "true".getBytes(UTF_8), CreateMode.EPHEMERAL, true);+ validateName(filePath, true);+ if (coreContainer.getPackageLoader().getPackageAPI().isJarInuse(filePath)) {+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "jar in use, can't delete");+ }+ FileStore.FileType type = fileStore.getType(filePath, true);+ if (type == FileStore.FileType.NOFILE) {+ throw new SolrException(+ SolrException.ErrorCode.BAD_REQUEST, "Path does not exist: " + filePath);+ }+ fileStore.delete(filePath);+ return response;+ } catch (SolrException e) {+ throw e;+ } catch (Exception e) {+ log.error("Unknown error", e);+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);+ } finally {+ try {+ coreContainer.getZkController().getZkClient().delete(TMP_ZK_NODE, -1, true);+ } catch (Exception e) {+ log.error("Unexpected error ", e);+ }+ }+ }++ private List<String> readSignatures(List<String> signatures, byte[] buf)+ throws SolrException, IOException {+ if (signatures == null || signatures.isEmpty()) return null;+ fileStore.refresh(KEYS_DIR);+ validate(signatures, buf);+ return signatures;+ }++ private void validate(List<String> sigs, byte[] buf) throws SolrException, IOException {+ Map<String, byte[]> keys = fileStore.getKeys();+ if (keys == null || keys.isEmpty()) {+ throw new SolrException(+ SolrException.ErrorCode.BAD_REQUEST, "File store does not have any keys");+ }+ CryptoKeys cryptoKeys = null;+ try {+ cryptoKeys = new CryptoKeys(keys);+ } catch (Exception e) {+ throw new SolrException(+ SolrException.ErrorCode.SERVER_ERROR, "Error parsing public keys in file store");+ }+ for (String sig : sigs) {+ if (cryptoKeys.verify(sig, ByteBuffer.wrap(buf)) == null) {+ throw new SolrException(+ SolrException.ErrorCode.BAD_REQUEST,+ "Signature does not match any public key : "+ + sig+ + " len: "+ + buf.length+ + " content sha512: "+ + DigestUtils.sha512Hex(buf));+ }+ }+ }++ /**+ * Creates a JSON string with the metadata.+ *+ * @lucene.internal+ */+ public static FileStoreAPI.MetaData _createJsonMetaData(byte[] buf, List<String> signatures)+ throws IOException {+ String sha512 = DigestUtils.sha512Hex(buf);+ Map<String, Object> vals = new HashMap<>();+ vals.put(FileStoreAPI.MetaData.SHA512, sha512);+ if (signatures != null) {+ vals.put("sig", signatures);+ }+ return new FileStoreAPI.MetaData(vals);+ }++ static final String INVALIDCHARS = " /\\#&*\n\t%@~`=+^$><?{}[]|:;!";++ public static void validateName(String path, boolean failForTrusted) {+ if (path == null) {+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "empty path");+ }+ List<String> parts = StrUtils.splitSmart(path, '/', true);+ for (String part : parts) {+ if (part.charAt(0) == '.') {+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "cannot start with period");+ }+ for (int i = 0; i < part.length(); i++) {+ for (int j = 0; j < INVALIDCHARS.length(); j++) {+ if (part.charAt(i) == INVALIDCHARS.charAt(j))+ throw new SolrException(+ SolrException.ErrorCode.BAD_REQUEST, "Unsupported char in file name: " + part);+ }+ }+ }+ if (failForTrusted && TRUSTED_DIR.equals(parts.get(0))) {+ throw new SolrException(+ SolrException.ErrorCode.BAD_REQUEST, "trying to write into /_trusted_/ directory");+ }+ }+}diff --git a/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java b/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.javaindex 4c62c1ff20e..7eb73354e6e 100644--- a/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java+++ b/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java@@ -216,8 +216,9 @@ public class DistribFileStore implements FileStore {}boolean fetchFromAnyNode() {- ArrayList<String> l = coreContainer.getFileStoreAPI().shuffledNodes();- for (String liveNode : l) {+ ArrayList<String> nodesToAttemptFetchFrom =+ FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer);+ for (String liveNode : nodesToAttemptFetchFrom) {try {String baseUrl =coreContainer.getZkController().getZkStateReader().getBaseUrlV2ForNodeName(liveNode);@@ -244,12 +245,6 @@ public class DistribFileStore implements FileStore {return false;}- String getSimpleName() {- int idx = path.lastIndexOf('/');- if (idx == -1) return path;- return path.substring(idx + 1);- }-public Path realPath() {return getRealpath(path);}@@ -306,6 +301,13 @@ public class DistribFileStore implements FileStore {ew.put("timestamp", getTimeStamp());if (metaData != null) metaData.writeMap(ew);}++ @Override+ public String getSimpleName() {+ int idx = path.lastIndexOf('/');+ if (idx == -1) return path;+ return path.substring(idx + 1);+ }};}@@ -353,7 +355,7 @@ public class DistribFileStore implements FileStore {}tmpFiles.put(info.path, info);- List<String> nodes = coreContainer.getFileStoreAPI().shuffledNodes();+ List<String> nodes = FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer);int i = 0;int FETCHFROM_SRC = 50;String myNodeName = coreContainer.getZkController().getNodeName();@@ -496,7 +498,7 @@ public class DistribFileStore implements FileStore {@Overridepublic void delete(String path) {deleteLocal(path);- List<String> nodes = coreContainer.getFileStoreAPI().shuffledNodes();+ List<String> nodes = FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer);HttpClient client = coreContainer.getUpdateShardHandler().getDefaultHttpClient();for (String node : nodes) {String baseUrl =@@ -583,7 +585,7 @@ public class DistribFileStore implements FileStore {}public static synchronized Path getFileStoreDirPath(Path solrHome) {- var path = solrHome.resolve(FileStoreAPI.FILESTORE_DIRECTORY);+ var path = solrHome.resolve(ClusterFileStore.FILESTORE_DIRECTORY);if (!Files.exists(path)) {try {Files.createDirectories(path);@@ -632,7 +634,7 @@ public class DistribFileStore implements FileStore {// reads local keys fileprivate static Map<String, byte[]> _getKeys(Path solrHome) throws IOException {Map<String, byte[]> result = new HashMap<>();- Path keysDir = _getRealPath(FileStoreAPI.KEYS_DIR, solrHome);+ Path keysDir = _getRealPath(ClusterFileStore.KEYS_DIR, solrHome);File[] keyFiles = keysDir.toFile().listFiles();if (keyFiles == null) return result;diff --git a/solr/core/src/java/org/apache/solr/filestore/FileStore.java b/solr/core/src/java/org/apache/solr/filestore/FileStore.javaindex fdff79e1fa5..65673757438 100644--- a/solr/core/src/java/org/apache/solr/filestore/FileStore.java+++ b/solr/core/src/java/org/apache/solr/filestore/FileStore.java@@ -111,6 +111,8 @@ public interface FileStore {interface FileDetails extends MapWriter {+ String getSimpleName();+MetaData getMetaData();Date getTimeStamp();diff --git a/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java b/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.javaindex dc420d37b31..d54acc464f5 100644--- a/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java+++ b/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java@@ -23,18 +23,11 @@ import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;import java.io.IOException;import java.io.InputStream;import java.lang.invoke.MethodHandles;-import java.nio.ByteBuffer;-import java.util.ArrayList;-import java.util.Arrays;import java.util.Collections;-import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.Objects;-import java.util.Set;-import java.util.function.Consumer;import java.util.function.Supplier;-import org.apache.commons.codec.digest.DigestUtils;import org.apache.solr.api.EndPoint;import org.apache.solr.client.solrj.SolrRequest;import org.apache.solr.common.MapWriter;@@ -42,18 +35,14 @@ import org.apache.solr.common.SolrException;import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.params.SolrParams;-import org.apache.solr.common.util.ContentStream;import org.apache.solr.common.util.StrUtils;import org.apache.solr.common.util.Utils;import org.apache.solr.core.CoreContainer;import org.apache.solr.core.SolrCore;-import org.apache.solr.pkg.PackageAPI;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.response.SolrQueryResponse;import org.apache.solr.security.PermissionNameProvider;import org.apache.solr.util.CryptoKeys;-import org.apache.zookeeper.CreateMode;-import org.apache.zookeeper.KeeperException;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -70,101 +59,11 @@ public class FileStoreAPI {public FileStoreAPI(CoreContainer coreContainer) {this.coreContainer = coreContainer;- fileStore = new DistribFileStore(coreContainer);- }-- public FileStore getFileStore() {- return fileStore;- }-- /** get a list of nodes randomly shuffled * @lucene.internal */- public ArrayList<String> shuffledNodes() {- Set<String> liveNodes =- coreContainer.getZkController().getZkStateReader().getClusterState().getLiveNodes();- ArrayList<String> l = new ArrayList<>(liveNodes);- l.remove(coreContainer.getZkController().getNodeName());- Collections.shuffle(l, Utils.RANDOM);- return l;- }-- public void validateFiles(List<String> files, boolean validateSignatures, Consumer<String> errs) {- for (String path : files) {- try {- FileStore.FileType type = fileStore.getType(path, true);- if (type != FileStore.FileType.FILE) {- errs.accept("No such file: " + path);- continue;- }-- fileStore.get(- path,- entry -> {- if (entry.getMetaData().signatures == null- || entry.getMetaData().signatures.isEmpty()) {- errs.accept(path + " has no signature");- return;- }- if (validateSignatures) {- try {- fileStore.refresh(KEYS_DIR);- validate(entry.meta.signatures, entry, false);- } catch (Exception e) {- log.error("Error validating package artifact", e);- errs.accept(e.getMessage());- }- }- },- false);- } catch (Exception e) {- log.error("Error reading file ", e);- errs.accept("Error reading file " + path + " " + e.getMessage());- }- }+ fileStore = coreContainer.getFileStore();}public class FSWrite {- static final String TMP_ZK_NODE = "/fileStoreWriteInProgress";-- @EndPoint(- path = "/cluster/files/*",- method = SolrRequest.METHOD.DELETE,- permission = PermissionNameProvider.Name.FILESTORE_WRITE_PERM)- public void delete(SolrQueryRequest req, SolrQueryResponse rsp) {- if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) {- throw new RuntimeException(PackageAPI.ERR_MSG);- }-- try {- coreContainer- .getZkController()- .getZkClient()- .create(TMP_ZK_NODE, "true".getBytes(UTF_8), CreateMode.EPHEMERAL, true);- String path = req.getPathTemplateValues().get("*");- validateName(path, true);- if (coreContainer.getPackageLoader().getPackageAPI().isJarInuse(path)) {- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "jar in use, can't delete");- }- FileStore.FileType type = fileStore.getType(path, true);- if (type == FileStore.FileType.NOFILE) {- throw new SolrException(- SolrException.ErrorCode.BAD_REQUEST, "Path does not exist: " + path);- }- fileStore.delete(path);- } catch (SolrException e) {- throw e;- } catch (Exception e) {- log.error("Unknown error", e);- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);- } finally {- try {- coreContainer.getZkController().getZkClient().delete(TMP_ZK_NODE, -1, true);- } catch (Exception e) {- log.error("Unexpected error ", e);- }- }- }-@EndPoint(path = "/node/files/*",method = SolrRequest.METHOD.DELETE,@@ -174,129 +73,6 @@ public class FileStoreAPI {validateName(path, true);fileStore.deleteLocal(path);}-- @EndPoint(- path = "/cluster/files/*",- method = SolrRequest.METHOD.PUT,- permission = PermissionNameProvider.Name.FILESTORE_WRITE_PERM)- public void upload(SolrQueryRequest req, SolrQueryResponse rsp) {- if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) {- throw new RuntimeException(PackageAPI.ERR_MSG);- }- try {- coreContainer- .getZkController()- .getZkClient()- .create(TMP_ZK_NODE, "true".getBytes(UTF_8), CreateMode.EPHEMERAL, true);-- Iterable<ContentStream> streams = req.getContentStreams();- if (streams == null)- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "no payload");- String path = req.getPathTemplateValues().get("*");- if (path == null) {- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No path");- }- validateName(path, true);- ContentStream stream = streams.iterator().next();- try {- byte[] buf = stream.getStream().readAllBytes();- List<String> signatures = readSignatures(req, buf);- MetaData meta = _createJsonMetaData(buf, signatures);- FileStore.FileType type = fileStore.getType(path, true);- boolean[] returnAfter = new boolean[] {false};- if (type == FileStore.FileType.FILE) {- // a file already exist at the same path- fileStore.get(- path,- fileEntry -> {- if (meta.equals(fileEntry.meta)) {- // the file content is same too. this is an idempotent put- // do not throw an error- returnAfter[0] = true;- rsp.add(CommonParams.FILE, path);- rsp.add("message", "File with same metadata exists ");- }- },- true);- }- if (returnAfter[0]) return;- if (type != FileStore.FileType.NOFILE) {- throw new SolrException(- SolrException.ErrorCode.BAD_REQUEST, "Path already exists " + path);- }- fileStore.put(new FileStore.FileEntry(ByteBuffer.wrap(buf), meta, path));- rsp.add(CommonParams.FILE, path);- } catch (IOException e) {- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);- }- } catch (InterruptedException e) {- log.error("Unexpected error", e);- } catch (KeeperException.NodeExistsException e) {- throw new SolrException(- SolrException.ErrorCode.SERVER_ERROR, "A write is already in process , try later");- } catch (KeeperException e) {- log.error("Unexpected error", e);- } finally {- try {- coreContainer.getZkController().getZkClient().delete(TMP_ZK_NODE, -1, true);- } catch (Exception e) {- log.error("Unexpected error ", e);- }- }- }-- private List<String> readSignatures(SolrQueryRequest req, byte[] buf)- throws SolrException, IOException {- String[] signatures = req.getParams().getParams("sig");- if (signatures == null || signatures.length == 0) return null;- List<String> sigs = Arrays.asList(signatures);- fileStore.refresh(KEYS_DIR);- validate(sigs, buf);- return sigs;- }-- private void validate(List<String> sigs, byte[] buf) throws SolrException, IOException {- Map<String, byte[]> keys = fileStore.getKeys();- if (keys == null || keys.isEmpty()) {- throw new SolrException(- SolrException.ErrorCode.BAD_REQUEST, "File store does not have any keys");- }- CryptoKeys cryptoKeys = null;- try {- cryptoKeys = new CryptoKeys(keys);- } catch (Exception e) {- throw new SolrException(- SolrException.ErrorCode.SERVER_ERROR, "Error parsing public keys in file store");- }- for (String sig : sigs) {- if (cryptoKeys.verify(sig, ByteBuffer.wrap(buf)) == null) {- throw new SolrException(- SolrException.ErrorCode.BAD_REQUEST,- "Signature does not match any public key : "- + sig- + " len: "- + buf.length- + " content sha512: "- + DigestUtils.sha512Hex(buf));- }- }- }- }-- /**- * Creates a JSON string with the metadata.- *- * @lucene.internal- */- public static MetaData _createJsonMetaData(byte[] buf, List<String> signatures)- throws IOException {- String sha512 = DigestUtils.sha512Hex(buf);- Map<String, Object> vals = new HashMap<>();- vals.put(MetaData.SHA512, sha512);- if (signatures != null) {- vals.put("sig", signatures);- }- return new MetaData(vals);}public class FSRead {diff --git a/solr/core/src/java/org/apache/solr/filestore/FileStoreUtils.java b/solr/core/src/java/org/apache/solr/filestore/FileStoreUtils.javanew file mode 100644index 00000000000..b9bd841e803--- /dev/null+++ b/solr/core/src/java/org/apache/solr/filestore/FileStoreUtils.java@@ -0,0 +1,145 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.filestore;++import java.io.IOException;+import java.io.InputStream;+import java.lang.invoke.MethodHandles;+import java.util.ArrayList;+import java.util.Collections;+import java.util.List;+import java.util.Map;+import java.util.Set;+import java.util.function.Consumer;+import java.util.function.Supplier;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.util.Utils;+import org.apache.solr.core.CoreContainer;+import org.apache.solr.util.CryptoKeys;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++/** Common utilities used by filestore-related code. */+public class FileStoreUtils {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ /** Returns a shuffled list of all live nodes except the current host */+ public static ArrayList<String> fetchAndShuffleRemoteLiveNodes(CoreContainer coreContainer) {+ Set<String> liveNodes =+ coreContainer.getZkController().getZkStateReader().getClusterState().getLiveNodes();+ ArrayList<String> l = new ArrayList<>(liveNodes);+ l.remove(coreContainer.getZkController().getNodeName());+ Collections.shuffle(l, Utils.RANDOM);+ return l;+ }++ public static void validateFiles(+ FileStore fileStore, List<String> files, boolean validateSignatures, Consumer<String> errs) {+ for (String path : files) {+ try {+ FileStore.FileType type = fileStore.getType(path, true);+ if (type != FileStore.FileType.FILE) {+ errs.accept("No such file: " + path);+ continue;+ }++ fileStore.get(+ path,+ entry -> {+ if (entry.getMetaData().signatures == null+ || entry.getMetaData().signatures.isEmpty()) {+ errs.accept(path + " has no signature");+ return;+ }+ if (validateSignatures) {+ try {+ fileStore.refresh(ClusterFileStore.KEYS_DIR);+ validate(fileStore, entry.meta.signatures, entry, false);+ } catch (Exception e) {+ log.error("Error validating package artifact", e);+ errs.accept(e.getMessage());+ }+ }+ },+ false);+ } catch (Exception e) {+ log.error("Error reading file ", e);+ errs.accept("Error reading file " + path + " " + e.getMessage());+ }+ }+ }++ /**+ * Validate a file for signature+ *+ * @param sigs the signatures, at least one should succeed+ * @param entry The file details+ * @param isFirstAttempt If there is a failure+ */+ public static void validate(+ FileStore fileStore, List<String> sigs, FileStore.FileEntry entry, boolean isFirstAttempt)+ throws SolrException, IOException {+ if (!isFirstAttempt) {+ // we are retrying because last validation failed.+ // get all keys again and try again+ fileStore.refresh(ClusterFileStore.KEYS_DIR);+ }++ Map<String, byte[]> keys = fileStore.getKeys();+ if (keys == null || keys.isEmpty()) {+ if (isFirstAttempt) {+ validate(fileStore, sigs, entry, false);+ return;+ }+ throw new SolrException(+ SolrException.ErrorCode.BAD_REQUEST, "Filestore does not have any public keys");+ }+ CryptoKeys cryptoKeys = null;+ try {+ cryptoKeys = new CryptoKeys(keys);+ } catch (Exception e) {+ throw new SolrException(+ SolrException.ErrorCode.SERVER_ERROR, "Error parsing public keys in ZooKeeper");+ }+ for (String sig : sigs) {+ Supplier<String> errMsg =+ () ->+ "Signature does not match any public key : "+ + sig+ + "sha256 "+ + entry.getMetaData().sha512;+ if (entry.getBuffer() != null) {+ if (cryptoKeys.verify(sig, entry.getBuffer()) == null) {+ if (isFirstAttempt) {+ validate(fileStore, sigs, entry, false);+ return;+ }+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.get());+ }+ } else {+ InputStream inputStream = entry.getInputStream();+ if (cryptoKeys.verify(sig, inputStream) == null) {+ if (isFirstAttempt) {+ validate(fileStore, sigs, entry, false);+ return;+ }+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.get());+ }+ }+ }+ }+}diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.javaindex c0d50dd85e2..d05c541243d 100644--- a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java+++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java@@ -48,6 +48,7 @@ import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.Utils;+import org.apache.solr.filestore.ClusterFileStore;import org.apache.solr.filestore.DistribFileStore;import org.apache.solr.filestore.FileStoreAPI;import org.apache.solr.packagemanager.SolrPackage.Manifest;@@ -301,7 +302,7 @@ public class PackageUtils {}public static void uploadKey(byte[] bytes, String path, Path home) throws IOException {- FileStoreAPI.MetaData meta = FileStoreAPI._createJsonMetaData(bytes, null);+ FileStoreAPI.MetaData meta = ClusterFileStore._createJsonMetaData(bytes, null);DistribFileStore._persistToFile(home, path, ByteBuffer.wrap(bytes), ByteBuffer.wrap(Utils.toJSON(meta)));}diff --git a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.javaindex d92d027d1c0..4e3be7b16c6 100644--- a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java+++ b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java@@ -51,7 +51,7 @@ import org.apache.solr.common.cloud.SolrZkClient;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.Utils;-import org.apache.solr.filestore.FileStoreAPI;+import org.apache.solr.filestore.ClusterFileStore;import org.apache.solr.packagemanager.SolrPackage.Artifact;import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease;import org.apache.solr.pkg.PackageAPI;@@ -146,7 +146,7 @@ public class RepositoryManager {String solrHome = (String) systemInfo.get("solr_home");// put the public key into package store's trusted key store and request a sync.- String path = FileStoreAPI.KEYS_DIR + "/" + destinationKeyFilename;+ String path = ClusterFileStore.KEYS_DIR + "/" + destinationKeyFilename;PackageUtils.uploadKey(key, path, Paths.get(solrHome));PackageUtils.getJsonStringFromNonCollectionApi(solrClient, "/api/node/files" + path, new ModifiableSolrParams().add("sync", "true"));diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.javaindex 7e2c5a6f73e..24339feefe8 100644--- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java+++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java@@ -45,7 +45,7 @@ import org.apache.solr.common.util.EnvUtils;import org.apache.solr.common.util.ReflectMapWriter;import org.apache.solr.common.util.Utils;import org.apache.solr.core.CoreContainer;-import org.apache.solr.filestore.FileStoreAPI;+import org.apache.solr.filestore.FileStoreUtils;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.response.SolrQueryResponse;import org.apache.solr.util.SolrJacksonAnnotationInspector;@@ -73,10 +73,6 @@ public class PackageAPI {public final Read readAPI = new Read();public PackageAPI(CoreContainer coreContainer, SolrPackageLoader loader) {- if (coreContainer.getFileStoreAPI() == null) {- throw new IllegalStateException("Must successfully load FileStoreAPI first");- }-this.coreContainer = coreContainer;this.packageLoader = loader;pkgs = new Packages();@@ -254,10 +250,10 @@ public class PackageAPI {}// first refresh my ownpackageLoader.notifyListeners(p);- for (String s : coreContainer.getFileStoreAPI().shuffledNodes()) {+ for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) {Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(),- coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(s)+ coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode)+ "/cluster/package?wt=javabin&omitHeader=true&refreshPackage="+ p,Utils.JAVABINCONSUMER);@@ -272,8 +268,8 @@ public class PackageAPI {payload.addError("No files specified");return;}- FileStoreAPI fileStoreAPI = coreContainer.getFileStoreAPI();- fileStoreAPI.validateFiles(add.files, true, s -> payload.addError(s));+ FileStoreUtils.validateFiles(+ coreContainer.getFileStore(), add.files, true, s -> payload.addError(s));if (payload.hasError()) return;Packages[] finalState = new Packages[1];try {@@ -422,10 +418,10 @@ public class PackageAPI {}void notifyAllNodesToSync(int expected) {- for (String s : coreContainer.getFileStoreAPI().shuffledNodes()) {+ for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) {Utils.executeGET(coreContainer.getUpdateShardHandler().getDefaultHttpClient(),- coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(s)+ coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode)+ "/cluster/package?wt=javabin&omitHeader=true&expectedVersion"+ expected,Utils.JAVABINCONSUMER);diff --git a/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.javaindex 5fe62e1f46b..0d5294f1ae6 100644--- a/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java+++ b/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java@@ -42,6 +42,7 @@ import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.core.CoreContainer;import org.apache.solr.core.SolrCore;import org.apache.solr.core.SolrResourceLoader;+import org.apache.solr.filestore.FileStoreUtils;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -272,12 +273,13 @@ public class SolrPackageLoader implements Closeable {List<Path> paths = new ArrayList<>();List<String> errs = new ArrayList<>();- coreContainer.getFileStoreAPI().validateFiles(version.files, true, s -> errs.add(s));+ FileStoreUtils.validateFiles(+ coreContainer.getFileStore(), version.files, true, s -> errs.add(s));if (!errs.isEmpty()) {throw new RuntimeException("Cannot load package: " + errs);}for (String file : version.files) {- paths.add(coreContainer.getFileStoreAPI().getFileStore().getRealpath(file));+ paths.add(coreContainer.getFileStore().getRealpath(file));}loader =diff --git a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.javaindex 62668789d22..f7d9144d812 100644--- a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java+++ b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java@@ -83,9 +83,8 @@ public class TestDistribFileStore extends SolrCloudTestCase {.addConfig("conf", configset("cloud-minimal")).configure();try {-byte[] derFile = readFile("cryptokeys/pub_key512.der");- uploadKey(derFile, FileStoreAPI.KEYS_DIR + "/pub_key512.der", cluster);+ uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster);try {postFile(diff --git a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.javaindex bd19c67f935..3b657e9cf72 100644--- a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java+++ b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java@@ -53,7 +53,7 @@ import org.apache.solr.common.util.ReflectMapWriter;import org.apache.solr.core.CoreContainer;import org.apache.solr.core.SolrResourceLoader;import org.apache.solr.embedded.JettySolrRunner;-import org.apache.solr.filestore.FileStoreAPI;+import org.apache.solr.filestore.ClusterFileStore;import org.apache.solr.filestore.TestDistribFileStore;import org.apache.solr.filestore.TestDistribFileStore.Fetcher;import org.apache.solr.pkg.PackageAPI;@@ -294,7 +294,7 @@ public class TestContainerPlugin extends SolrCloudTestCase {int version = phaser.getPhase();byte[] derFile = readFile("cryptokeys/pub_key512.der");- uploadKey(derFile, FileStoreAPI.KEYS_DIR + "/pub_key512.der", cluster);+ uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster);TestPackages.postFileAndWait(cluster,"runtimecode/containerplugin.v.1.jar.bin",diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.javaindex aae36019a65..04423132451 100644--- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java+++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java@@ -68,7 +68,7 @@ import org.apache.solr.common.util.ReflectMapWriter;import org.apache.solr.common.util.Utils;import org.apache.solr.core.SolrCore;import org.apache.solr.embedded.JettySolrRunner;-import org.apache.solr.filestore.FileStoreAPI;+import org.apache.solr.filestore.ClusterFileStore;import org.apache.solr.filestore.TestDistribFileStore;import org.apache.solr.handler.RequestHandlerBase;import org.apache.solr.request.SolrQueryRequest;@@ -121,7 +121,7 @@ public class TestPackages extends SolrCloudTestCase {String FILE1 = "/mypkg/runtimelibs.jar";String COLLECTION_NAME = "testCoreReloadingPluginColl";byte[] derFile = readFile("cryptokeys/pub_key512.der");- uploadKey(derFile, FileStoreAPI.KEYS_DIR + "/pub_key512.der", cluster);+ uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster);postFileAndWait(cluster,"runtimecode/runtimelibs.jar.bin",@@ -190,7 +190,7 @@ public class TestPackages extends SolrCloudTestCase {String EXPR1 = "/mypkg/expressible.jar";String COLLECTION_NAME = "testPluginLoadingColl";byte[] derFile = readFile("cryptokeys/pub_key512.der");- uploadKey(derFile, FileStoreAPI.KEYS_DIR + "/pub_key512.der", cluster);+ uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster);postFileAndWait(cluster,"runtimecode/runtimelibs.jar.bin",@@ -609,7 +609,7 @@ public class TestPackages extends SolrCloudTestCase {expectError(req, cluster.getSolrClient(), errPath, FILE1 + " has no signature");// now we upload the keysbyte[] derFile = readFile("cryptokeys/pub_key512.der");- uploadKey(derFile, FileStoreAPI.KEYS_DIR + "/pub_key512.der", cluster);+ uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster);// and upload the same file with a different name, but it has proper signaturepostFileAndWait(cluster,@@ -755,7 +755,7 @@ public class TestPackages extends SolrCloudTestCase {String FILE1 = "/schemapkg/schema-plugins.jar";byte[] derFile = readFile("cryptokeys/pub_key512.der");- uploadKey(derFile, FileStoreAPI.KEYS_DIR + "/pub_key512.der", cluster);+ uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster);postFileAndWait(cluster,"runtimecode/schema-plugins.jar.bin",diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/response/SolrResponseBase.java b/solr/solrj/src/java/org/apache/solr/client/solrj/response/SolrResponseBase.javaindex 85818b78a98..9d90184ce42 100644--- a/solr/solrj/src/java/org/apache/solr/client/solrj/response/SolrResponseBase.java+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/response/SolrResponseBase.java@@ -17,6 +17,7 @@package org.apache.solr.client.solrj.response;import java.io.IOException;+import java.util.Map;import org.apache.solr.client.solrj.SolrResponse;import org.apache.solr.common.MapWriter;import org.apache.solr.common.util.NamedList;@@ -59,8 +60,32 @@ public class SolrResponseBase extends SolrResponse implements MapWriter {return response.toString();}+ /**+ * Return a {@link NamedList} object representing the 'responseHeader' section of Solr's response+ *+ * <p>This method may return null, if no responseHeader can be found. If a value is returned, it+ * should not be modified. Any modifications made are not guaranteed to be durable.+ */+ @SuppressWarnings("unchecked")public NamedList<?> getResponseHeader() {- return (NamedList<?>) response.get("responseHeader");+ // ResponseParser implementations vary in what types they use when deserializing responses, so+ // we need to be a bit flexible in inspecting types within the NamedList. See SOLR-17316 for+ // details.+ final var responseHeader = response.get("responseHeader");+ if (responseHeader == null) {+ return null;+ }++ if (responseHeader instanceof NamedList) {+ return (NamedList<?>) responseHeader;+ } else if (responseHeader instanceof Map) {+ final var responseHeaderAsMap = (Map<String, Object>) responseHeader;+ return new NamedList<>(responseHeaderAsMap);+ }+ throw new IllegalStateException(+ "'responseHeader' key was an unexpected type ["+ + responseHeader.getClass().getSimpleName()+ + "]");}// these two methods are based on the logic in SolrCore.setResponseHeaderValues(...)diff --git a/solr/solrj/src/resources/java-template/api.mustache b/solr/solrj/src/resources/java-template/api.mustacheindex 95fc0397188..7fe5bdf6324 100644--- a/solr/solrj/src/resources/java-template/api.mustache+++ b/solr/solrj/src/resources/java-template/api.mustache@@ -17,6 +17,9 @@package {{package}};+import java.io.IOException;+import java.io.InputStream;+import java.io.OutputStream;import java.lang.reflect.Type;import java.util.ArrayList;import java.util.HashMap;@@ -156,6 +159,19 @@ public class {{classname}} {{{#vendorExtensions.x-genericEntity}}if (requestBody instanceof String) {return new RequestWriter.StringPayloadContentWriter((String) requestBody, expectedType);+ } else if (requestBody instanceof InputStream) {+ final var inputStream = (InputStream) requestBody;+ return new RequestWriter.ContentWriter() {+ @Override+ public void write(OutputStream os) throws IOException {+ inputStream.transferTo(os);+ }++ @Override+ public String getContentType() {+ return expectedType;+ }+ };}{{/vendorExtensions.x-genericEntity}}return new JacksonContentWriter(expectedType, requestBody);