Apache Solr open-source search software
Revert "SOLR-17351: Decompose filestore "get file" API (#3047)"
Jason Gerlowskidd6e18e54ed3
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 7c6bc431711..277691d7ad5 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -37,10 +37,6 @@ Improvements* SOLR-17516: `LBHttp2SolrClient` is now generic, adding support for `HttpJdkSolrClient`. (James Dyer)-* SOLR-17351: Solr's filestore "get-file" API has been decomposed into several separate endpoints. Traditional file-fetching is now available at- `GET /api/cluster/filestore/files/some/path.txt` and metadata fetching (and directory listing) is now available at `GET /api/cluster/filestore/metadata/some/path.txt`.- SolrJ now offers request and response bindings for these APIs in `org.apache.solr.client.solrj.request.FileStoreApi`. (Jason Gerlowski)-Optimizations---------------------* SOLR-17568: The CLI bin/solr export tool now contacts the appropriate nodes directly for data instead of proxying through one.diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.javaindex 2445b17e7aa..11ee4e1e26d 100644--- a/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java+++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java@@ -17,7 +17,6 @@package org.apache.solr.client.api.endpoint;import static org.apache.solr.client.api.util.Constants.GENERIC_ENTITY_PROPERTY;-import static org.apache.solr.client.api.util.Constants.RAW_OUTPUT_PROPERTY;import io.swagger.v3.oas.annotations.Operation;import io.swagger.v3.oas.annotations.Parameter;@@ -25,19 +24,16 @@ import io.swagger.v3.oas.annotations.extensions.Extension;import io.swagger.v3.oas.annotations.extensions.ExtensionProperty;import io.swagger.v3.oas.annotations.parameters.RequestBody;import jakarta.ws.rs.DELETE;-import jakarta.ws.rs.GET;-import jakarta.ws.rs.POST;import jakarta.ws.rs.PUT;import jakarta.ws.rs.Path;import jakarta.ws.rs.PathParam;import jakarta.ws.rs.QueryParam;import java.io.InputStream;import java.util.List;-import org.apache.solr.client.api.model.FileStoreDirectoryListingResponse;import org.apache.solr.client.api.model.SolrJerseyResponse;import org.apache.solr.client.api.model.UploadToFileStoreResponse;-@Path("/cluster/filestore")+@Path("/cluster")public interface ClusterFileStoreApis {// TODO Better understand the purpose of the 'sig' parameter and improve docs here.@PUT@@ -60,29 +56,6 @@ public interface ClusterFileStoreApis {})InputStream requestBody);- @GET- @Operation(- summary = "Retrieve metadata about a file or directory in the filestore.",- tags = {"file-store"})- @Path("/metadata{path:.+}")- FileStoreDirectoryListingResponse getMetadata(- @Parameter(description = "Path to a file or directory within the filestore")- @PathParam("path")- String path);-- @GET- @Operation(- summary = "Retrieve raw contents of a file in the filestore.",- tags = {"file-store"},- extensions = {- @Extension(properties = {@ExtensionProperty(name = RAW_OUTPUT_PROPERTY, value = "true")})- })- @Path("/files{filePath:.+}")- SolrJerseyResponse getFile(- @Parameter(description = "Path to a file or directory within the filestore")- @PathParam("filePath")- String path);-@DELETE@Operation(summary = "Delete a file or directory from the filestore.",@@ -97,27 +70,4 @@ public interface ClusterFileStoreApis {"Indicates whether the deletion should only be done on the receiving node. For internal use only")@QueryParam("localDelete")Boolean localDelete);-- @POST- @Operation(- summary = "Fetches a filestore entry from other nodes in the cluster.",- tags = {"file-store"})- @Path("/commands/fetch{path:.+}")- SolrJerseyResponse fetchFile(- @Parameter(description = "Path to a file or directory within the filestore")- @PathParam("path")- String path,- @Parameter(description = "An optional Solr node name to fetch the file from")- @QueryParam("getFrom")- String getFrom);-- @POST- @Operation(- summary = "Syncs a file by pushing it to other nodes in the cluster.",- tags = {"file-store"})- @Path("/commands/sync{path:.+}")- SolrJerseyResponse syncFile(- @Parameter(description = "Path to a file or directory within the filestore")- @PathParam("path")- String path);}diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeFileStoreApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeFileStoreApis.javanew file mode 100644index 00000000000..15f4a73cfb1--- /dev/null+++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeFileStoreApis.java@@ -0,0 +1,67 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.client.api.endpoint;++import static org.apache.solr.client.api.util.Constants.OMIT_FROM_CODEGEN_PROPERTY;++import io.swagger.v3.oas.annotations.Operation;+import io.swagger.v3.oas.annotations.Parameter;+import io.swagger.v3.oas.annotations.extensions.Extension;+import io.swagger.v3.oas.annotations.extensions.ExtensionProperty;+import jakarta.ws.rs.GET;+import jakarta.ws.rs.Path;+import jakarta.ws.rs.PathParam;+import jakarta.ws.rs.QueryParam;+import org.apache.solr.client.api.model.SolrJerseyResponse;++/**+ * V2 APIs for fetching filestore files, syncing them across nodes, or fetching related metadata.+ */+@Path("/node")+public interface NodeFileStoreApis {+ @GET+ @Operation(+ summary = "Retrieve file contents or metadata from the filestore.",+ tags = {"file-store"},+ // The response of this v2 API is highly variable based on the parameters specified. It can+ // return raw (potentially binary) file data, a JSON-ified representation of that file data,+ // metadata regarding one or multiple file store entries, etc. This variability can be+ // handled on the Jersey server side, but would be prohibitively difficult to accommodate in+ // our code-generation templates. Ideally, cosmetic improvements (e.g. splitting it up into+ // multiple endpoints) will make this unnecessary in the future. But for now, the extension+ // property below ensures that this endpoint is ignored entirely when doing code generation.+ extensions = {+ @Extension(+ properties = {@ExtensionProperty(name = OMIT_FROM_CODEGEN_PROPERTY, value = "true")})+ })+ @Path("/files{path:.+}")+ SolrJerseyResponse getFile(+ @Parameter(description = "Path to a file or directory within the filestore")+ @PathParam("path")+ String path,+ @Parameter(+ description =+ "If true, triggers syncing for this file across all nodes in the filestore")+ @QueryParam("sync")+ Boolean sync,+ @Parameter(description = "An optional Solr node name to fetch the file from")+ @QueryParam("getFrom")+ String getFrom,+ @Parameter(description = "Indicates that (only) file metadata should be fetched")+ @QueryParam("meta")+ Boolean meta);+}diff --git a/solr/api/src/java/org/apache/solr/client/api/model/FileStoreDirectoryListingResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/FileStoreDirectoryListingResponse.javaindex 79e002b6cee..bcbc5f1f728 100644--- a/solr/api/src/java/org/apache/solr/client/api/model/FileStoreDirectoryListingResponse.java+++ b/solr/api/src/java/org/apache/solr/client/api/model/FileStoreDirectoryListingResponse.java@@ -19,6 +19,10 @@ package org.apache.solr.client.api.model;import com.fasterxml.jackson.annotation.JsonProperty;import java.util.Map;+/**+ * One of several possible responses from {@link+ * org.apache.solr.client.api.endpoint.NodeFileStoreApis#getFile(String, Boolean, String, Boolean)}+ */public class FileStoreDirectoryListingResponse extends SolrJerseyResponse {@JsonProperty public Map<String, Object> files;}diff --git a/solr/api/src/java/org/apache/solr/client/api/model/FileStoreJsonFileResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/FileStoreJsonFileResponse.javanew file mode 100644index 00000000000..321faec01ec--- /dev/null+++ b/solr/api/src/java/org/apache/solr/client/api/model/FileStoreJsonFileResponse.java@@ -0,0 +1,30 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.client.api.model;++import com.fasterxml.jackson.annotation.JsonProperty;++/**+ * One of several possible responses from {@link+ * org.apache.solr.client.api.endpoint.NodeFileStoreApis#getFile(String, Boolean, String, Boolean)}+ *+ * <p>Typically used when 'wt=json' is specified while retrieving an individual file from the+ * filestore+ */+public class FileStoreJsonFileResponse extends SolrJerseyResponse {+ @JsonProperty public String response;+}diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.javaindex 1ed72a71a7a..e791b3ca602 100644--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java@@ -110,6 +110,7 @@ import org.apache.solr.core.backup.repository.BackupRepositoryFactory;import org.apache.solr.filestore.ClusterFileStore;import org.apache.solr.filestore.DistribFileStore;import org.apache.solr.filestore.FileStore;+import org.apache.solr.filestore.NodeFileStore;import org.apache.solr.handler.ClusterAPI;import org.apache.solr.handler.RequestHandlerBase;import org.apache.solr.handler.SnapShooter;@@ -879,6 +880,7 @@ public class CoreContainer {fileStore = new DistribFileStore(this);registerV2ApiIfEnabled(ClusterFileStore.class);+ registerV2ApiIfEnabled(NodeFileStore.class);packageLoader = new SolrPackageLoader(this);registerV2ApiIfEnabled(packageLoader.getPackageAPI().editAPI);diff --git a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.javaindex 043192aa0d8..db422baf6bf 100644--- a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java+++ b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java@@ -18,33 +18,23 @@package org.apache.solr.filestore;import static java.nio.charset.StandardCharsets.UTF_8;-import static org.apache.solr.handler.admin.api.ReplicationAPIBase.FILE_STREAM;-import static org.apache.solr.response.RawResponseWriter.CONTENT;import jakarta.inject.Inject;import java.io.IOException;import java.io.InputStream;import java.lang.invoke.MethodHandles;import java.nio.ByteBuffer;-import java.util.Collections;import java.util.HashMap;import java.util.List;import java.util.Map;-import java.util.stream.Collectors;import org.apache.commons.codec.digest.DigestUtils;import org.apache.solr.api.JerseyResource;import org.apache.solr.client.api.endpoint.ClusterFileStoreApis;-import org.apache.solr.client.api.model.FileStoreDirectoryListingResponse;-import org.apache.solr.client.api.model.FileStoreEntryMetadata;import org.apache.solr.client.api.model.SolrJerseyResponse;import org.apache.solr.client.api.model.UploadToFileStoreResponse;import org.apache.solr.common.SolrException;-import org.apache.solr.common.params.CommonParams;-import org.apache.solr.common.params.ModifiableSolrParams;-import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.StrUtils;import org.apache.solr.core.CoreContainer;-import org.apache.solr.core.SolrCore;import org.apache.solr.jersey.PermissionName;import org.apache.solr.pkg.PackageAPI;import org.apache.solr.request.SolrQueryRequest;@@ -151,115 +141,6 @@ public class ClusterFileStore extends JerseyResource implements ClusterFileStorereturn response;}- @Override- @PermissionName(PermissionNameProvider.Name.FILESTORE_READ_PERM)- public SolrJerseyResponse getFile(String path) {- final var response = instantiateJerseyResponse(SolrJerseyResponse.class);-- final var type = fileStore.getType(path, false);- if (type == FileStore.FileType.NOFILE) {- throw new SolrException(- SolrException.ErrorCode.NOT_FOUND,- "Requested path [" + path + "] not found in filestore");- } else if (type == FileStore.FileType.DIRECTORY) {- throw new SolrException(- SolrException.ErrorCode.BAD_REQUEST,- "Requested path [" + path + "] is a directory and has no returnable contents");- }-- attachFileToResponse(path, fileStore, req, rsp);- return response;- }-- @Override- @PermissionName(PermissionNameProvider.Name.FILESTORE_READ_PERM)- public FileStoreDirectoryListingResponse getMetadata(String path) {- if (path == null) {- path = "";- }- FileStore.FileType type = fileStore.getType(path, false);- return getMetadata(type, path, fileStore);- }-- public static void attachFileToResponse(- String path, FileStore fileStore, SolrQueryRequest req, SolrQueryResponse rsp) {- ModifiableSolrParams solrParams = new ModifiableSolrParams();- solrParams.add(CommonParams.WT, FILE_STREAM);- req.setParams(SolrParams.wrapDefaults(solrParams, req.getParams()));- rsp.add(- CONTENT,- (SolrCore.RawWriter)- os ->- fileStore.get(- path,- it -> {- try {- InputStream inputStream = it.getInputStream();- if (inputStream != null) {- inputStream.transferTo(os);- }- } catch (IOException e) {- throw new SolrException(- SolrException.ErrorCode.SERVER_ERROR, "Error reading file " + path);- }- },- false));- }-- @SuppressWarnings("fallthrough")- public static FileStoreDirectoryListingResponse getMetadata(- FileStore.FileType type, String path, FileStore fileStore) {- final var dirListingResponse = new FileStoreDirectoryListingResponse();- if (path == null) {- path = "";- }-- switch (type) {- case NOFILE:- dirListingResponse.files = Collections.singletonMap(path, null);- break;- case METADATA:- case FILE:- int idx = path.lastIndexOf('/');- String fileName = path.substring(idx + 1);- String parentPath = path.substring(0, path.lastIndexOf('/'));- List<FileStore.FileDetails> l = fileStore.list(parentPath, s -> s.equals(fileName));-- dirListingResponse.files =- Collections.singletonMap(path, l.isEmpty() ? null : convertToResponse(l.get(0)));- break;- case DIRECTORY:- final var directoryContents =- fileStore.list(path, null).stream()- .map(details -> convertToResponse(details))- .collect(Collectors.toList());- dirListingResponse.files = Collections.singletonMap(path, directoryContents);- break;- }-- return dirListingResponse;- }-- // TODO Modify the filestore implementation itself to return this object, so conversion isn't- // needed.- private static FileStoreEntryMetadata convertToResponse(FileStore.FileDetails details) {- final var entryMetadata = new FileStoreEntryMetadata();-- entryMetadata.name = details.getSimpleName();- if (details.isDir()) {- entryMetadata.dir = true;- return entryMetadata;- }-- entryMetadata.size = details.size();- entryMetadata.timestamp = details.getTimeStamp();- if (details.getMetaData() != null) {- details.getMetaData().toMap(entryMetadata.unknownProperties());- }-- return entryMetadata;- }-private void doLocalDelete(String filePath) {fileStore.deleteLocal(filePath);}@@ -313,50 +194,6 @@ public class ClusterFileStore extends JerseyResource implements ClusterFileStorereturn response;}- @Override- @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM)- public SolrJerseyResponse fetchFile(String path, String getFrom) {- final var response = instantiateJerseyResponse(SolrJerseyResponse.class);- if (path == null) {- path = "";- }- pullFileFromNode(coreContainer, fileStore, path, getFrom);- return response;- }-- @Override- @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM)- public SolrJerseyResponse syncFile(String path) {- final var response = instantiateJerseyResponse(SolrJerseyResponse.class);- syncToAllNodes(fileStore, path);- return response;- }-- public static void syncToAllNodes(FileStore fileStore, String path) {- try {- fileStore.syncToAllNodes(path);- } catch (IOException e) {- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error getting file ", e);- }- }-- public static void pullFileFromNode(- CoreContainer coreContainer, FileStore fileStore, String path, String getFrom) {- coreContainer- .getUpdateShardHandler()- .getUpdateExecutor()- .submit(- () -> {- log.debug("Downloading file {}", path);- try {- fileStore.fetch(path, getFrom);- } catch (Exception e) {- log.error("Failed to download file: {}", path, e);- }- log.info("downloaded file: {}", path);- });- }-private List<String> readSignatures(List<String> signatures, byte[] buf)throws SolrException, IOException {if (signatures == null || signatures.isEmpty()) return null;diff --git a/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java b/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.javaindex 42679fd51ce..00e71968250 100644--- a/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java+++ b/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java@@ -18,6 +18,8 @@package org.apache.solr.filestore;import static java.nio.charset.StandardCharsets.UTF_8;+import static org.apache.solr.client.solrj.SolrRequest.METHOD.DELETE;+import static org.apache.solr.client.solrj.SolrRequest.METHOD.GET;import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST;import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;@@ -34,6 +36,7 @@ import java.nio.file.Path;import java.nio.file.Paths;import java.nio.file.StandardOpenOption;import java.util.ArrayList;+import java.util.Arrays;import java.util.Date;import java.util.HashMap;import java.util.List;@@ -47,9 +50,11 @@ import net.jcip.annotations.NotThreadSafe;import org.apache.commons.codec.digest.DigestUtils;import org.apache.lucene.util.IOUtils;import org.apache.solr.client.solrj.SolrServerException;-import org.apache.solr.client.solrj.request.FileStoreApi;+import org.apache.solr.client.solrj.impl.InputStreamResponseParser;+import org.apache.solr.client.solrj.request.GenericSolrRequest;import org.apache.solr.common.SolrException;import org.apache.solr.common.cloud.SolrZkClient;+import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.util.Utils;import org.apache.solr.core.CoreContainer;import org.apache.solr.core.SolrPaths;@@ -187,31 +192,26 @@ public class DistribFileStore implements FileStore {var solrClient = coreContainer.getDefaultHttpSolrClient();try {- final var metadataRequest = new FileStoreApi.GetFile(getMetaPath());- final var client = coreContainer.getSolrClientCache().getHttpSolrClient(baseUrl);- final var response = metadataRequest.process(client);+ GenericSolrRequest request = new GenericSolrRequest(GET, "/node/files" + getMetaPath());+ request.setResponseParser(new InputStreamResponseParser(null));+ var response = solrClient.requestWithBaseUrl(baseUrl, request::process).getResponse();+ is = (InputStream) response.get("stream");metadata =- Utils.newBytesConsumer((int) MAX_PKG_SIZE)- .accept(response.getResponseStreamIfSuccessful());+ Utils.newBytesConsumer((int) MAX_PKG_SIZE).accept((InputStream) response.get("stream"));m = (Map<?, ?>) Utils.fromJSON(metadata.array(), metadata.arrayOffset(), metadata.limit());- } catch (Exception e) {+ } catch (SolrServerException | IOException e) {throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error fetching metadata", e);} finally {org.apache.solr.common.util.IOUtils.closeQuietly(is);}- ByteBuffer filedata = null;- try {- final var fileRequest = new FileStoreApi.GetFile(path);- final var fileResponse = solrClient.requestWithBaseUrl(baseUrl, null, fileRequest);- try (final var stream = fileResponse.getResponseStreamIfSuccessful()) {- filedata = Utils.newBytesConsumer((int) MAX_PKG_SIZE).accept(stream);- }- } catch (Exception e) {- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error fetching data", e);- }-try {+ GenericSolrRequest request = new GenericSolrRequest(GET, "/node/files" + path);+ request.setResponseParser(new InputStreamResponseParser(null));+ var response = solrClient.requestWithBaseUrl(baseUrl, request::process).getResponse();+ is = (InputStream) response.get("stream");+ ByteBuffer filedata =+ Utils.newBytesConsumer((int) MAX_PKG_SIZE).accept((InputStream) response.get("stream"));filedata.mark();String sha512 = DigestUtils.sha512Hex(new ByteBufferInputStream(filedata));String expected = (String) m.get("sha512");@@ -222,6 +222,8 @@ public class DistribFileStore implements FileStore {filedata.reset();persistToFile(filedata, metadata);return true;+ } catch (SolrServerException e) {+ throw new SolrException(SERVER_ERROR, "Error fetching data", e);} catch (IOException ioe) {throw new SolrException(SERVER_ERROR, "Error persisting file", ioe);} finally {@@ -236,12 +238,18 @@ public class DistribFileStore implements FileStore {try {String baseUrl =coreContainer.getZkController().getZkStateReader().getBaseUrlV2ForNodeName(liveNode);- final var metadataRequest = new FileStoreApi.GetMetadata(path);- final var client = coreContainer.getSolrClientCache().getHttpSolrClient(baseUrl);- final var metadataResponse = metadataRequest.process(client).getParsed();- boolean nodeHasBlob =- metadataResponse.files != null && metadataResponse.files.containsKey(path);+ final var solrParams = new ModifiableSolrParams();+ solrParams.add("meta", "true");+ solrParams.add("omitHeader", "true");+ final var request = new GenericSolrRequest(GET, "/node/files" + path, solrParams);+ boolean nodeHasBlob = false;+ var solrClient = coreContainer.getDefaultHttpSolrClient();+ var resp = solrClient.requestWithBaseUrl(baseUrl, request::process).getResponse();++ if (Utils.getObjectByPath(resp, false, Arrays.asList("files", path)) != null) {+ nodeHasBlob = true;+ }if (nodeHasBlob) {boolean success = fetchFileFromNodeAndPersist(liveNode);if (success) return true;@@ -359,13 +367,11 @@ public class DistribFileStore implements FileStore {for (String node : nodes) {String baseUrl =coreContainer.getZkController().getZkStateReader().getBaseUrlV2ForNodeName(node);-- String nodeToFetchFrom;if (i < FETCHFROM_SRC) {// this is to protect very large clusters from overwhelming a single node// the first FETCHFROM_SRC nodes will be asked to fetch from this node.// it's there in the memory now. So , it must be served fast- nodeToFetchFrom = myNodeName;+ getFrom = myNodeName;} else {if (i == FETCHFROM_SRC) {// This is just an optimization@@ -376,17 +382,19 @@ public class DistribFileStore implements FileStore {} catch (Exception e) {}}- // trying to avoid the thundering herd problem when there are a very large number of- // nodes others should try to fetch it from any node where it is available. By now,+ // trying to avoid the thundering herd problem when there are a very large no:of nodes+ // others should try to fetch it from any node where it is available. By now,// almost FETCHFROM_SRC other nodes may have it- nodeToFetchFrom = "*";+ getFrom = "*";}try {- final var pullFileRequest = new FileStoreApi.FetchFile(info.path);- pullFileRequest.setGetFrom(nodeToFetchFrom);- final var client = coreContainer.getSolrClientCache().getHttpSolrClient(baseUrl);+ var solrClient = coreContainer.getDefaultHttpSolrClient();+ var solrParams = new ModifiableSolrParams();+ solrParams.set("getFrom", getFrom);++ var request = new GenericSolrRequest(GET, "/node/files" + info.path, solrParams);// fire and forget- pullFileRequest.process(client);+ solrClient.requestWithBaseUrl(baseUrl, request::process);} catch (Exception e) {log.info("Node: {} failed to respond for file fetch notification", node, e);// ignore the exception@@ -502,8 +510,9 @@ public class DistribFileStore implements FileStore {deleteLocal(path);List<String> nodes = FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer);- final var solrRequest = new FileStoreApi.DeleteFile(path);- solrRequest.setLocalDelete(Boolean.TRUE);+ final var solrParams = new ModifiableSolrParams();+ solrParams.add("localDelete", "true");+ final var solrRequest = new GenericSolrRequest(DELETE, "/cluster/files" + path, solrParams);for (String node : nodes) {String baseUrl =diff --git a/solr/core/src/java/org/apache/solr/filestore/NodeFileStore.java b/solr/core/src/java/org/apache/solr/filestore/NodeFileStore.javanew file mode 100644index 00000000000..e6c61419437--- /dev/null+++ b/solr/core/src/java/org/apache/solr/filestore/NodeFileStore.java@@ -0,0 +1,210 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.filestore;++import static java.nio.charset.StandardCharsets.UTF_8;+import static org.apache.solr.handler.admin.api.ReplicationAPIBase.FILE_STREAM;+import static org.apache.solr.response.RawResponseWriter.CONTENT;+import static org.apache.solr.security.PermissionNameProvider.Name.FILESTORE_READ_PERM;++import jakarta.inject.Inject;+import java.io.IOException;+import java.io.InputStream;+import java.lang.invoke.MethodHandles;+import java.util.Collections;+import java.util.List;+import java.util.stream.Collectors;+import org.apache.solr.api.JerseyResource;+import org.apache.solr.client.api.endpoint.NodeFileStoreApis;+import org.apache.solr.client.api.model.FileStoreDirectoryListingResponse;+import org.apache.solr.client.api.model.FileStoreEntryMetadata;+import org.apache.solr.client.api.model.FileStoreJsonFileResponse;+import org.apache.solr.client.api.model.SolrJerseyResponse;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.params.CommonParams;+import org.apache.solr.common.params.ModifiableSolrParams;+import org.apache.solr.common.params.SolrParams;+import org.apache.solr.core.CoreContainer;+import org.apache.solr.core.SolrCore;+import org.apache.solr.jersey.PermissionName;+import org.apache.solr.request.SolrQueryRequest;+import org.apache.solr.response.SolrQueryResponse;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++/** Implementation for {@link NodeFileStoreApis} */+public class NodeFileStore extends JerseyResource implements NodeFileStoreApis {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ private final CoreContainer coreContainer;+ private final SolrQueryRequest req;+ private final SolrQueryResponse rsp;+ private final FileStore fileStore;++ @Inject+ public NodeFileStore(+ CoreContainer coreContainer,+ DistribFileStore fileStore,+ SolrQueryRequest req,+ SolrQueryResponse rsp) {+ this.coreContainer = coreContainer;+ this.req = req;+ this.rsp = rsp;+ this.fileStore = fileStore;+ }++ // TODO SOLR-17351 - this single "get" operation actually supports several different chunks of+ // functionality: syncing, directory listing, file-fetching, metadata-fetching. We should split+ // it up into multiple distinct APIs+ @Override+ @PermissionName(FILESTORE_READ_PERM)+ public SolrJerseyResponse getFile(String path, Boolean sync, String getFrom, Boolean meta) {+ final var response = instantiateJerseyResponse(SolrJerseyResponse.class);++ if (Boolean.TRUE.equals(sync)) {+ try {+ fileStore.syncToAllNodes(path);+ return response;+ } catch (IOException e) {+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error getting file ", e);+ }+ }++ if (path == null) {+ path = "";+ }+ final var pathCopy = path;+ if (getFrom != null) {+ coreContainer+ .getUpdateShardHandler()+ .getUpdateExecutor()+ .submit(+ () -> {+ log.debug("Downloading file {}", pathCopy);+ try {+ fileStore.fetch(pathCopy, getFrom);+ } catch (Exception e) {+ log.error("Failed to download file: {}", pathCopy, e);+ }+ log.info("downloaded file: {}", pathCopy);+ });+ return response;+ }++ FileStore.FileType type = fileStore.getType(path, false);+ if (type == FileStore.FileType.NOFILE) {+ final var fileMissingResponse =+ instantiateJerseyResponse(FileStoreDirectoryListingResponse.class);+ fileMissingResponse.files = Collections.singletonMap(path, null);+ return fileMissingResponse;+ }+ if (type == FileStore.FileType.DIRECTORY) {+ final var directoryListingResponse =+ instantiateJerseyResponse(FileStoreDirectoryListingResponse.class);+ final var directoryContents =+ fileStore.list(path, null).stream()+ .map(details -> convertToResponse(details))+ .collect(Collectors.toList());+ directoryListingResponse.files = Collections.singletonMap(path, directoryContents);+ return directoryListingResponse;+ }+ if (Boolean.TRUE.equals(meta)) {+ if (type == FileStore.FileType.FILE) {+ int idx = path.lastIndexOf('/');+ String fileName = path.substring(idx + 1);+ String parentPath = path.substring(0, path.lastIndexOf('/'));+ List<FileStore.FileDetails> l = fileStore.list(parentPath, s -> s.equals(fileName));++ final var fileMetaResponse =+ instantiateJerseyResponse(FileStoreDirectoryListingResponse.class);+ fileMetaResponse.files =+ Collections.singletonMap(path, l.isEmpty() ? null : convertToResponse(l.get(0)));+ return fileMetaResponse;+ }+ } else { // User wants to get the "raw" file+ // TODO Should we be trying to json-ify otherwise "raw" files in this way? It seems like a+ // pretty sketchy idea, esp. for code with very little test coverage. Consider removing+ if ("json".equals(req.getParams().get(CommonParams.WT))) {+ final var jsonResponse = instantiateJerseyResponse(FileStoreJsonFileResponse.class);+ try {+ fileStore.get(+ pathCopy,+ it -> {+ try {+ InputStream inputStream = it.getInputStream();+ if (inputStream != null) {+ jsonResponse.response = new String(inputStream.readAllBytes(), UTF_8);+ }+ } catch (IOException e) {+ throw new SolrException(+ SolrException.ErrorCode.SERVER_ERROR, "Error reading file " + pathCopy);+ }+ },+ false);+ return jsonResponse;+ } catch (IOException e) {+ throw new SolrException(+ SolrException.ErrorCode.SERVER_ERROR, "Error getting file from path " + path);+ }+ } else {+ ModifiableSolrParams solrParams = new ModifiableSolrParams();+ solrParams.add(CommonParams.WT, FILE_STREAM);+ req.setParams(SolrParams.wrapDefaults(solrParams, req.getParams()));+ rsp.add(+ CONTENT,+ (SolrCore.RawWriter)+ os ->+ fileStore.get(+ pathCopy,+ it -> {+ try {+ InputStream inputStream = it.getInputStream();+ if (inputStream != null) {+ inputStream.transferTo(os);+ }+ } catch (IOException e) {+ throw new SolrException(+ SolrException.ErrorCode.SERVER_ERROR,+ "Error reading file " + pathCopy);+ }+ },+ false));+ }+ }+ return response;+ }++ // TODO Modify the filestore implementation itself to return this object, so conversion isn't+ // needed.+ private FileStoreEntryMetadata convertToResponse(FileStore.FileDetails details) {+ final var entryMetadata = new FileStoreEntryMetadata();++ entryMetadata.name = details.getSimpleName();+ if (details.isDir()) {+ entryMetadata.dir = true;+ return entryMetadata;+ }++ entryMetadata.size = details.size();+ entryMetadata.timestamp = details.getTimeStamp();+ if (details.getMetaData() != null) {+ details.getMetaData().toMap(entryMetadata.unknownProperties());+ }++ return entryMetadata;+ }+}diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.javaindex 07fb72965eb..1d69a9c429d 100644--- a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java+++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java@@ -173,7 +173,7 @@ public class PackageManager implements Closeable {String.format(Locale.ROOT, "/package/%s/%s/%s", packageName, version, "manifest.json"));for (String filePath : filesToDelete) {DistribFileStore.deleteZKFileEntry(zkClient, filePath);- String path = "/api/cluster/filestore/files" + filePath;+ String path = "/api/cluster/files" + filePath;printGreen("Deleting " + path);solrClient.request(new GenericSolrRequest(SolrRequest.METHOD.DELETE, path));}diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.javaindex b0bb80cef42..1d1270a2b84 100644--- a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java+++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java@@ -32,7 +32,6 @@ import java.util.List;import java.util.Map;import java.util.zip.ZipEntry;import java.util.zip.ZipFile;-import org.apache.commons.io.IOUtils;import org.apache.lucene.util.SuppressForbidden;import org.apache.solr.cli.CLIUtils;import org.apache.solr.client.solrj.SolrClient;@@ -41,6 +40,7 @@ import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.impl.JsonMapResponseParser;import org.apache.solr.client.solrj.request.FileStoreApi;import org.apache.solr.client.solrj.request.GenericSolrRequest;+import org.apache.solr.client.solrj.request.GenericV2SolrRequest;import org.apache.solr.common.SolrException;import org.apache.solr.common.SolrException.ErrorCode;import org.apache.solr.common.params.ModifiableSolrParams;@@ -182,27 +182,26 @@ public class PackageUtils {public static Manifest fetchManifest(SolrClient solrClient, String manifestFilePath, String expectedSHA512)throws IOException, SolrServerException {-- final var manifestRequest = new FileStoreApi.GetFile(manifestFilePath);- final var manifestResponse = manifestRequest.process(solrClient);- try (final var manifestStream = manifestResponse.getResponseStreamIfSuccessful()) {- final var manifestJson = IOUtils.toString(manifestStream, StandardCharsets.UTF_8);- String calculatedSHA512 =- Utils.sha512Digest(ByteBuffer.wrap(manifestJson.getBytes(StandardCharsets.UTF_8)));- if (!expectedSHA512.equals(calculatedSHA512)) {- throw new SolrException(- ErrorCode.UNAUTHORIZED,- "The manifest SHA512 doesn't match expected SHA512. Possible unauthorized manipulation. "- + "Expected: "- + expectedSHA512- + ", calculated: "- + calculatedSHA512- + ", manifest location: "- + manifestFilePath);- }- Manifest manifest = getMapper().readValue(manifestJson, Manifest.class);- return manifest;+ GenericSolrRequest request =+ new GenericV2SolrRequest(SolrRequest.METHOD.GET, "/api/node/files" + manifestFilePath);+ request.setResponseParser(new JsonMapResponseParser());+ NamedList<Object> response = solrClient.request(request);+ String manifestJson = (String) response.get("response");+ String calculatedSHA512 =+ Utils.sha512Digest(ByteBuffer.wrap(manifestJson.getBytes(StandardCharsets.UTF_8)));+ if (expectedSHA512.equals(calculatedSHA512) == false) {+ throw new SolrException(+ ErrorCode.UNAUTHORIZED,+ "The manifest SHA512 doesn't match expected SHA512. Possible unauthorized manipulation. "+ + "Expected: "+ + expectedSHA512+ + ", calculated: "+ + calculatedSHA512+ + ", manifest location: "+ + manifestFilePath);}+ Manifest manifest = getMapper().readValue(manifestJson, Manifest.class);+ return manifest;}/**diff --git a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.javaindex 7c5306ceabc..c855f52961e 100644--- a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java+++ b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java@@ -44,7 +44,6 @@ import org.apache.solr.client.api.util.SolrVersion;import org.apache.solr.client.solrj.SolrClient;import org.apache.solr.client.solrj.SolrRequest;import org.apache.solr.client.solrj.SolrServerException;-import org.apache.solr.client.solrj.request.FileStoreApi;import org.apache.solr.client.solrj.request.GenericSolrRequest;import org.apache.solr.client.solrj.request.GenericV2SolrRequest;import org.apache.solr.client.solrj.request.RequestWriter;@@ -52,6 +51,7 @@ import org.apache.solr.client.solrj.request.beans.PackagePayload;import org.apache.solr.common.SolrException;import org.apache.solr.common.SolrException.ErrorCode;import org.apache.solr.common.cloud.SolrZkClient;+import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.Utils;import org.apache.solr.filestore.ClusterFileStore;@@ -151,14 +151,8 @@ public class RepositoryManager {// put the public key into package store's trusted key store and request a sync.String path = ClusterFileStore.KEYS_DIR + "/" + destinationKeyFilename;PackageUtils.uploadKey(key, path, Paths.get(solrHome));- final var syncRequest = new FileStoreApi.SyncFile(path);- final var syncResponse = syncRequest.process(solrClient).getParsed();- final var status = syncResponse.responseHeader.status;- if (status != 0) {- throw new SolrException(- ErrorCode.getErrorCode(status),- "Unexpected status " + status + " while syncing filestore upload.");- }+ PackageUtils.getJsonStringFromNonCollectionApi(+ solrClient, "/api/node/files" + path, new ModifiableSolrParams().add("sync", "true"));}private String getRepositoriesJson(SolrZkClient zkClient)diff --git a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.javaindex 95732ea3a1c..ff8f68b7897 100644--- a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java+++ b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java@@ -40,7 +40,6 @@ import org.apache.solr.client.solrj.SolrRequest;import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.impl.BaseHttpSolrClient.RemoteExecutionException;import org.apache.solr.client.solrj.impl.HttpSolrClient;-import org.apache.solr.client.solrj.request.FileStoreApi;import org.apache.solr.client.solrj.request.V2Request;import org.apache.solr.client.solrj.response.SimpleSolrResponse;import org.apache.solr.client.solrj.response.V2Response;@@ -116,7 +115,7 @@ public class TestDistribFileStore extends SolrCloudTestCase {assertResponseValues(10,cluster.getSolrClient(),- new V2Request.Builder("/cluster/filestore/metadata/package/mypkg/v1.0")+ new V2Request.Builder("/node/files/package/mypkg/v1.0").withMethod(SolrRequest.METHOD.GET).build(),Map.of(@@ -129,7 +128,7 @@ public class TestDistribFileStore extends SolrCloudTestCase {assertResponseValues(10,cluster.getSolrClient(),- new V2Request.Builder("/cluster/filestore/metadata/package/mypkg")+ new V2Request.Builder("/node/files/package/mypkg").withMethod(SolrRequest.METHOD.GET).build(),Map.of(@@ -170,15 +169,14 @@ public class TestDistribFileStore extends SolrCloudTestCase {});for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) {String baseUrl = jettySolrRunner.getBaseURLV2().toString();- String url = baseUrl + "/cluster/filestore/metadata/package/mypkg/v1.0?wt=javabin";+ String url = baseUrl + "/node/files/package/mypkg/v1.0?wt=javabin";assertResponseValues(10, new Fetcher(url, jettySolrRunner), expected);}// Delete JarsDistribFileStore.deleteZKFileEntry(cluster.getZkClient(), "/package/mypkg/v1.0/runtimelibs.jar");JettySolrRunner j = cluster.getRandomJetty(random());- String path =- j.getBaseURLV2() + "/cluster/filestore/files" + "/package/mypkg/v1.0/runtimelibs.jar";+ String path = j.getBaseURLV2() + "/cluster/files" + "/package/mypkg/v1.0/runtimelibs.jar";HttpDelete del = new HttpDelete(path);try (HttpSolrClient cl = (HttpSolrClient) j.newClient()) {Utils.executeHttpMethod(cl.getHttpClient(), path, Utils.JSONCONSUMER, del);@@ -198,7 +196,7 @@ public class TestDistribFileStore extends SolrCloudTestCase {throws Exception {for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) {String baseUrl = jettySolrRunner.getBaseURLV2().toString();- String url = baseUrl + "/cluster/filestore/metadata" + path + "?wt=javabin";+ String url = baseUrl + "/node/files" + path + "?wt=javabin&meta=true";assertResponseValues(10, new Fetcher(url, jettySolrRunner), expected);if (verifyContent) {@@ -206,7 +204,7 @@ public class TestDistribFileStore extends SolrCloudTestCase {ByteBuffer buf =Utils.executeGET(solrClient.getHttpClient(),- baseUrl + "/cluster/filestore/files" + path,+ baseUrl + "/node/files" + path,Utils.newBytesConsumer(Integer.MAX_VALUE));assertEquals("d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420",@@ -321,12 +319,9 @@ public class TestDistribFileStore extends SolrCloudTestCase {JettySolrRunner jetty = cluster.getRandomJetty(random());try (HttpSolrClient client = (HttpSolrClient) jetty.newClient()) {PackageUtils.uploadKey(bytes, path, Paths.get(jetty.getCoreContainer().getSolrHome()));-- final var syncReq = new FileStoreApi.SyncFile(path);- final var syncRsp = syncReq.process(client);- if (log.isInfoEnabled()) {- log.info("sync resp for path {} was {}", path, syncRsp.getParsed().responseHeader.status);- }+ String url = jetty.getBaseURLV2() + "/node/files" + path + "?sync=true";+ Object resp = Utils.executeGET(client.getHttpClient(), url, null);+ log.info("sync resp: {} was {}", url, resp);}checkAllNodesForFile(cluster,@@ -338,7 +333,7 @@ public class TestDistribFileStore extends SolrCloudTestCase {public static NavigableObject postFile(SolrClient client, ByteBuffer buffer, String name, String sig)throws SolrServerException, IOException {- String resource = "/cluster/filestore/files" + name;+ String resource = "/cluster/files" + name;ModifiableSolrParams params = new ModifiableSolrParams();params.add("sig", sig);V2Response rsp =diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java b/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.javaindex 5fa62780a6b..26d8261c07f 100644--- a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java+++ b/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java@@ -124,7 +124,7 @@ public class PackageStoreSchemaPluginsTest extends SolrCloudTestCase {private void uploadPluginJar(String version, Path jarPath) throws Exception {var pluginRequest =- new V2Request.Builder("/cluster/filestore/files/my-plugin/plugin-" + version + ".jar")+ new V2Request.Builder("/cluster/files/my-plugin/plugin-" + version + ".jar").PUT().withParams(params("sig", signature(Files.readAllBytes(jarPath)))).withPayload(Files.newInputStream(jarPath))diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adocindex 33b6afcc2a4..df8ed1f3907 100644--- a/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc+++ b/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc@@ -88,9 +88,10 @@ The metadata file stores the sha512 and signatures of the jar files too.The end points are:-* `PUT /api/cluster/filestore/files/{full/path/to/file}` to upload a new file and add it to the file store-* `GET /api/cluster/filestore/files/{full/path/to/file}` to download a file already in the filestore-* `GET /api/cluster/filestore/metadata/{full/path/to/file}` to get the metadata of a particular file, or a list of files available in a directory+* `PUT /api/cluster/files/{full/path/to/file}` in each node+* `GET /api/node/files/{full/path/to/file}` to download the file+* `GET /api/node/files/{full/path/to/file}?meta=true` get the metadata of the file+* `GET /api/node/files/{full/path/to/}` get the list of files in `/full/path/to`=== Signing Your Artifacts@@ -114,14 +115,14 @@ openssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64 | sed+[source, bash]-----curl --data-binary @runtimelibs.jar -X PUT http://localhost:8983/api/cluster/filestore/files/mypkg/1.0/myplugins.jar?sig=<signature-of-jar>+curl --data-binary @runtimelibs.jar -X PUT http://localhost:8983/api/cluster/files/mypkg/1.0/myplugins.jar?sig=<signature-of-jar>----. Verify your jar upload:+[source, bash]-----curl http://localhost:8983/api/cluster/filestore/metadata/mypkg/1.0?omitHeader=true+curl http://localhost:8983/api/node/files/mypkg/1.0?omitHeader=true----+[source, json]@@ -327,14 +328,14 @@ curl -o runtimelibs3.jar -LO https://github.com/apache/solr/blob/releases/solropenssl dgst -sha1 -sign my_key.pem runtimelibs.jar | openssl enc -base64 | sed 's/+/%2B/g' | tr -d \\n | sed-curl --data-binary @runtimelibs3.jar -X PUT http://localhost:8983/api/cluster/filestore/files/mypkg/2.0/myplugins.jar?sig=<signature>+curl --data-binary @runtimelibs3.jar -X PUT http://localhost:8983/api/cluster/files/mypkg/2.0/myplugins.jar?sig=<signature>----. Verify it:+[source, bash]-----curl http://localhost:8983/api/cluster/filestore/metadata/mypkg/2.0?omitHeader=true+curl http://localhost:8983/api/node/files/mypkg/2.0?omitHeader=true----+[source, json]diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/InputStreamResponse.java b/solr/solrj/src/java/org/apache/solr/client/solrj/InputStreamResponse.javaindex 6770ce81c53..b86e76f13ac 100644--- a/solr/solrj/src/java/org/apache/solr/client/solrj/InputStreamResponse.java+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/InputStreamResponse.java@@ -70,11 +70,12 @@ public class InputStreamResponse extends SimpleSolrResponse {/*** Access the server response as an {@link InputStream}, regardless of the HTTP status code*- * <p>Caller is responsible for consuming and closing the stream. No validation is done on the- * HTTP status code.+ * <p>Caller is responsible for consuming and closing the stream, and releasing it from the+ * tracking done by {@link ObjectReleaseTracker}. No validation is done on the HTTP status code.*/public InputStream getResponseStream() {final NamedList<Object> resp = getResponse();+return (InputStream) resp.get(STREAM_KEY);}@@ -123,6 +124,7 @@ public class InputStreamResponse extends SimpleSolrResponse {inputStream.transferTo(baos);return baos.toString(Charset.defaultCharset());} finally {+ ObjectReleaseTracker.release(inputStream);IOUtils.closeQuietly(baos);IOUtils.closeQuietly(inputStream);}diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.javaindex 7940b8ddce3..646ef546c99 100644--- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java@@ -97,8 +97,6 @@ public class ClientUtils {private static String buildReplacementV2Path(String existingPath) {if (existingPath.contains("/solr")) {return existingPath.replaceFirst("/solr", "/api");- } else if (existingPath.endsWith("/api")) {- return existingPath;} else {return existingPath + "/api";}diff --git a/solr/solrj/src/resources/java-template/api.mustache b/solr/solrj/src/resources/java-template/api.mustacheindex 0ed28650056..f276f3f0a72 100644--- a/solr/solrj/src/resources/java-template/api.mustache+++ b/solr/solrj/src/resources/java-template/api.mustache@@ -23,12 +23,10 @@ import java.io.OutputStream;import java.lang.reflect.Type;import java.util.ArrayList;import java.util.HashMap;-import java.util.HashSet;import java.util.List;import java.util.Map;import java.util.Set;-import org.apache.solr.client.solrj.SolrClient;import org.apache.solr.client.solrj.SolrRequest;import org.apache.solr.client.solrj.response.SimpleSolrResponse;@@ -241,15 +239,6 @@ public class {{classname}} {return params;}- @Override- public Set<String> getQueryParams() {- final var queryParams = new HashSet<String>();- {{#queryParams}}- queryParams.add("{{baseName}}");- {{/queryParams}}- return queryParams;- }-@Overrideprotected {{operationIdCamelCase}}Response createResponse(SolrClient client) {return new {{operationIdCamelCase}}Response();