Apache Solr open-source search software
SOLR-17302: Convert remaining filestore APIs to JAX-RS (#2532)
Jason Gerlowskid0aad69c0ffc
diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.javaindex 07746b5e92d..11ee4e1e26d 100644--- a/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java+++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java@@ -64,5 +64,10 @@ public interface ClusterFileStoreApis {SolrJerseyResponse deleteFile(@Parameter(description = "Path to a file or directory within the filestore")@PathParam("path")- String path);+ String path,+ @Parameter(+ description =+ "Indicates whether the deletion should only be done on the receiving node. For internal use only")+ @QueryParam("localDelete")+ Boolean localDelete);}diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeFileStoreApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeFileStoreApis.javanew file mode 100644index 00000000000..15f4a73cfb1--- /dev/null+++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeFileStoreApis.java@@ -0,0 +1,67 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.client.api.endpoint;++import static org.apache.solr.client.api.util.Constants.OMIT_FROM_CODEGEN_PROPERTY;++import io.swagger.v3.oas.annotations.Operation;+import io.swagger.v3.oas.annotations.Parameter;+import io.swagger.v3.oas.annotations.extensions.Extension;+import io.swagger.v3.oas.annotations.extensions.ExtensionProperty;+import jakarta.ws.rs.GET;+import jakarta.ws.rs.Path;+import jakarta.ws.rs.PathParam;+import jakarta.ws.rs.QueryParam;+import org.apache.solr.client.api.model.SolrJerseyResponse;++/**+ * V2 APIs for fetching filestore files, syncing them across nodes, or fetching related metadata.+ */+@Path("/node")+public interface NodeFileStoreApis {+ @GET+ @Operation(+ summary = "Retrieve file contents or metadata from the filestore.",+ tags = {"file-store"},+ // The response of this v2 API is highly variable based on the parameters specified. It can+ // return raw (potentially binary) file data, a JSON-ified representation of that file data,+ // metadata regarding one or multiple file store entries, etc. This variability can be+ // handled on the Jersey server side, but would be prohibitively difficult to accommodate in+ // our code-generation templates. Ideally, cosmetic improvements (e.g. splitting it up into+ // multiple endpoints) will make this unnecessary in the future. But for now, the extension+ // property below ensures that this endpoint is ignored entirely when doing code generation.+ extensions = {+ @Extension(+ properties = {@ExtensionProperty(name = OMIT_FROM_CODEGEN_PROPERTY, value = "true")})+ })+ @Path("/files{path:.+}")+ SolrJerseyResponse getFile(+ @Parameter(description = "Path to a file or directory within the filestore")+ @PathParam("path")+ String path,+ @Parameter(+ description =+ "If true, triggers syncing for this file across all nodes in the filestore")+ @QueryParam("sync")+ Boolean sync,+ @Parameter(description = "An optional Solr node name to fetch the file from")+ @QueryParam("getFrom")+ String getFrom,+ @Parameter(description = "Indicates that (only) file metadata should be fetched")+ @QueryParam("meta")+ Boolean meta);+}diff --git a/solr/api/src/java/org/apache/solr/client/api/model/FileStoreDirectoryListingResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/FileStoreDirectoryListingResponse.javanew file mode 100644index 00000000000..bcbc5f1f728--- /dev/null+++ b/solr/api/src/java/org/apache/solr/client/api/model/FileStoreDirectoryListingResponse.java@@ -0,0 +1,28 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.client.api.model;++import com.fasterxml.jackson.annotation.JsonProperty;+import java.util.Map;++/**+ * One of several possible responses from {@link+ * org.apache.solr.client.api.endpoint.NodeFileStoreApis#getFile(String, Boolean, String, Boolean)}+ */+public class FileStoreDirectoryListingResponse extends SolrJerseyResponse {+ @JsonProperty public Map<String, Object> files;+}diff --git a/solr/api/src/java/org/apache/solr/client/api/model/FileStoreEntryMetadata.java b/solr/api/src/java/org/apache/solr/client/api/model/FileStoreEntryMetadata.javanew file mode 100644index 00000000000..5a9bc213452--- /dev/null+++ b/solr/api/src/java/org/apache/solr/client/api/model/FileStoreEntryMetadata.java@@ -0,0 +1,44 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.client.api.model;++import com.fasterxml.jackson.annotation.JsonAnyGetter;+import com.fasterxml.jackson.annotation.JsonAnySetter;+import com.fasterxml.jackson.annotation.JsonProperty;+import java.util.Date;+import java.util.HashMap;+import java.util.Map;++/** Represents the metadata for a single filestore file or directory */+public class FileStoreEntryMetadata {+ @JsonProperty public String name;+ @JsonProperty public Boolean dir;+ @JsonProperty public Long size;+ @JsonProperty public Date timestamp;++ private Map<String, Object> additionalMetadata = new HashMap<>();++ @JsonAnyGetter+ public Map<String, Object> unknownProperties() {+ return additionalMetadata;+ }++ @JsonAnySetter+ public void setUnknownProperty(String field, Object value) {+ additionalMetadata.put(field, value);+ }+}diff --git a/solr/api/src/java/org/apache/solr/client/api/model/FileStoreJsonFileResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/FileStoreJsonFileResponse.javanew file mode 100644index 00000000000..321faec01ec--- /dev/null+++ b/solr/api/src/java/org/apache/solr/client/api/model/FileStoreJsonFileResponse.java@@ -0,0 +1,30 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.client.api.model;++import com.fasterxml.jackson.annotation.JsonProperty;++/**+ * One of several possible responses from {@link+ * org.apache.solr.client.api.endpoint.NodeFileStoreApis#getFile(String, Boolean, String, Boolean)}+ *+ * <p>Typically used when 'wt=json' is specified while retrieving an individual file from the+ * filestore+ */+public class FileStoreJsonFileResponse extends SolrJerseyResponse {+ @JsonProperty public String response;+}diff --git a/solr/api/src/java/org/apache/solr/client/api/util/Constants.java b/solr/api/src/java/org/apache/solr/client/api/util/Constants.javaindex 560ce77af97..b4ef56c2050 100644--- a/solr/api/src/java/org/apache/solr/client/api/util/Constants.java+++ b/solr/api/src/java/org/apache/solr/client/api/util/Constants.java@@ -27,6 +27,7 @@ public class Constants {public static final String INDEX_PATH_PREFIX ="/{" + INDEX_TYPE_PATH_PARAMETER + ":cores|collections}/{" + INDEX_NAME_PATH_PARAMETER + "}";+ public static final String OMIT_FROM_CODEGEN_PROPERTY = "omitFromCodegen";public static final String GENERIC_ENTITY_PROPERTY = "genericEntity";public static final String BINARY_CONTENT_TYPE_V2 = "application/vnd.apache.solr.javabin";diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.javaindex a05bfed4e7d..f14680d5561 100644--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java@@ -107,7 +107,7 @@ import org.apache.solr.core.backup.repository.BackupRepositoryFactory;import org.apache.solr.filestore.ClusterFileStore;import org.apache.solr.filestore.DistribFileStore;import org.apache.solr.filestore.FileStore;-import org.apache.solr.filestore.FileStoreAPI;+import org.apache.solr.filestore.NodeFileStore;import org.apache.solr.handler.ClusterAPI;import org.apache.solr.handler.RequestHandlerBase;import org.apache.solr.handler.SnapShooter;@@ -298,7 +298,6 @@ public class CoreContainer {private DelegatingPlacementPluginFactory placementPluginFactory;private DistribFileStore fileStore;- private FileStoreAPI fileStoreAPI;private ClusterFileStore clusterFileStoreAPI;private SolrPackageLoader packageLoader;@@ -862,10 +861,8 @@ public class CoreContainer {pkiAuthenticationSecurityBuilder.initializeMetrics(solrMetricsContext, "/authentication/pki");fileStore = new DistribFileStore(this);- fileStoreAPI = new FileStoreAPI(this);- registerV2ApiIfEnabled(fileStoreAPI.readAPI);- registerV2ApiIfEnabled(fileStoreAPI.writeAPI);registerV2ApiIfEnabled(ClusterFileStore.class);+ registerV2ApiIfEnabled(NodeFileStore.class);packageLoader = new SolrPackageLoader(this);registerV2ApiIfEnabled(packageLoader.getPackageAPI().editAPI);diff --git a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.javaindex a664e5699cb..db422baf6bf 100644--- a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java+++ b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java@@ -141,12 +141,15 @@ public class ClusterFileStore extends JerseyResource implements ClusterFileStorereturn response;}- @Override- @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM)- public SolrJerseyResponse deleteFile(String filePath) {- final var response = instantiateJerseyResponse(SolrJerseyResponse.class);- if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) {- throw new RuntimeException(PackageAPI.ERR_MSG);+ private void doLocalDelete(String filePath) {+ fileStore.deleteLocal(filePath);+ }++ private void doClusterDelete(String filePath) {+ FileStore.FileType type = fileStore.getType(filePath, true);+ if (type == FileStore.FileType.NOFILE) {+ throw new SolrException(+ SolrException.ErrorCode.BAD_REQUEST, "Path does not exist: " + filePath);}try {@@ -154,19 +157,7 @@ public class ClusterFileStore extends JerseyResource implements ClusterFileStore.getZkController().getZkClient().create(TMP_ZK_NODE, "true".getBytes(UTF_8), CreateMode.EPHEMERAL, true);- validateName(filePath, true);- if (coreContainer.getPackageLoader().getPackageAPI().isJarInuse(filePath)) {- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "jar in use, can't delete");- }- FileStore.FileType type = fileStore.getType(filePath, true);- if (type == FileStore.FileType.NOFILE) {- throw new SolrException(- SolrException.ErrorCode.BAD_REQUEST, "Path does not exist: " + filePath);- }fileStore.delete(filePath);- return response;- } catch (SolrException e) {- throw e;} catch (Exception e) {log.error("Unknown error", e);throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);@@ -179,6 +170,30 @@ public class ClusterFileStore extends JerseyResource implements ClusterFileStore}}+ private void doDelete(String filePath, Boolean localDelete) {+ if (Boolean.TRUE.equals(localDelete)) {+ doLocalDelete(filePath);+ } else {+ doClusterDelete(filePath);+ }+ }++ @Override+ @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM)+ public SolrJerseyResponse deleteFile(String filePath, Boolean localDelete) {+ final var response = instantiateJerseyResponse(SolrJerseyResponse.class);+ if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) {+ throw new RuntimeException(PackageAPI.ERR_MSG);+ }++ validateName(filePath, true);+ if (coreContainer.getPackageLoader().getPackageAPI().isJarInuse(filePath)) {+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "jar in use, can't delete");+ }+ doDelete(filePath, localDelete);+ return response;+ }+private List<String> readSignatures(List<String> signatures, byte[] buf)throws SolrException, IOException {if (signatures == null || signatures.isEmpty()) return null;diff --git a/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java b/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.javaindex 7eb73354e6e..1a462dee26e 100644--- a/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java+++ b/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java@@ -50,7 +50,6 @@ import org.apache.http.client.methods.HttpDelete;import org.apache.lucene.util.IOUtils;import org.apache.solr.common.SolrException;import org.apache.solr.common.cloud.SolrZkClient;-import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.util.Utils;import org.apache.solr.core.CoreContainer;import org.apache.solr.core.SolrPaths;@@ -288,20 +287,6 @@ public class DistribFileStore implements FileStore {return realPath().toFile().length();}- @Override- public void writeMap(EntryWriter ew) throws IOException {- MetaData metaData = readMetaData();- ew.put(CommonParams.NAME, getSimpleName());- if (type == FileType.DIRECTORY) {- ew.put("dir", true);- return;- }-- ew.put("size", size());- ew.put("timestamp", getTimeStamp());- if (metaData != null) metaData.writeMap(ew);- }-@Overridepublic String getSimpleName() {int idx = path.lastIndexOf('/');@@ -503,7 +488,7 @@ public class DistribFileStore implements FileStore {for (String node : nodes) {String baseUrl =coreContainer.getZkController().getZkStateReader().getBaseUrlV2ForNodeName(node);- String url = baseUrl + "/node/files" + path;+ String url = baseUrl + "/cluster/files" + path + "?localDelete=true";HttpDelete del = new HttpDelete(url);// invoke delete command on all nodes asynchronouslycoreContainer.runAsync(() -> Utils.executeHttpMethod(client, url, null, del));diff --git a/solr/core/src/java/org/apache/solr/filestore/FileStore.java b/solr/core/src/java/org/apache/solr/filestore/FileStore.javaindex 65673757438..916b8b33b3b 100644--- a/solr/core/src/java/org/apache/solr/filestore/FileStore.java+++ b/solr/core/src/java/org/apache/solr/filestore/FileStore.java@@ -26,7 +26,6 @@ import java.util.List;import java.util.Map;import java.util.function.Consumer;import java.util.function.Predicate;-import org.apache.solr.common.MapWriter;import org.apache.solr.filestore.FileStoreAPI.MetaData;import org.apache.zookeeper.server.ByteBufferInputStream;@@ -109,7 +108,7 @@ public interface FileStore {METADATA}- interface FileDetails extends MapWriter {+ interface FileDetails {String getSimpleName();diff --git a/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java b/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.javaindex d54acc464f5..14e8e1e0c34 100644--- a/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java+++ b/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java@@ -17,173 +17,16 @@package org.apache.solr.filestore;-import static java.nio.charset.StandardCharsets.UTF_8;-import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;-import java.io.IOException;-import java.io.InputStream;-import java.lang.invoke.MethodHandles;-import java.util.Collections;import java.util.List;import java.util.Map;import java.util.Objects;-import java.util.function.Supplier;-import org.apache.solr.api.EndPoint;-import org.apache.solr.client.solrj.SolrRequest;import org.apache.solr.common.MapWriter;-import org.apache.solr.common.SolrException;-import org.apache.solr.common.params.CommonParams;-import org.apache.solr.common.params.ModifiableSolrParams;-import org.apache.solr.common.params.SolrParams;-import org.apache.solr.common.util.StrUtils;import org.apache.solr.common.util.Utils;-import org.apache.solr.core.CoreContainer;-import org.apache.solr.core.SolrCore;-import org.apache.solr.request.SolrQueryRequest;-import org.apache.solr.response.SolrQueryResponse;-import org.apache.solr.security.PermissionNameProvider;-import org.apache.solr.util.CryptoKeys;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;public class FileStoreAPI {- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());- public static final String FILESTORE_DIRECTORY = "filestore";- public static final String TRUSTED_DIR = "_trusted_";public static final String KEYS_DIR = "/_trusted_/keys";- private final CoreContainer coreContainer;- FileStore fileStore;- public final FSRead readAPI = new FSRead();- public final FSWrite writeAPI = new FSWrite();-- public FileStoreAPI(CoreContainer coreContainer) {- this.coreContainer = coreContainer;- fileStore = coreContainer.getFileStore();- }-- public class FSWrite {-- @EndPoint(- path = "/node/files/*",- method = SolrRequest.METHOD.DELETE,- permission = PermissionNameProvider.Name.FILESTORE_WRITE_PERM)- public void deleteLocal(SolrQueryRequest req, SolrQueryResponse rsp) {- String path = req.getPathTemplateValues().get("*");- validateName(path, true);- fileStore.deleteLocal(path);- }- }-- public class FSRead {- @EndPoint(- path = "/node/files/*",- method = SolrRequest.METHOD.GET,- permission = PermissionNameProvider.Name.FILESTORE_READ_PERM)- public void read(SolrQueryRequest req, SolrQueryResponse rsp) {- String path = req.getPathTemplateValues().get("*");- String pathCopy = path;- if (req.getParams().getBool("sync", false)) {- try {- fileStore.syncToAllNodes(path);- return;- } catch (IOException e) {- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error getting file ", e);- }- }- String getFrom = req.getParams().get("getFrom");- if (getFrom != null) {- coreContainer- .getUpdateShardHandler()- .getUpdateExecutor()- .submit(- () -> {- log.debug("Downloading file {}", pathCopy);- try {- fileStore.fetch(pathCopy, getFrom);- } catch (Exception e) {- log.error("Failed to download file: {}", pathCopy, e);- }- log.info("downloaded file: {}", pathCopy);- });- return;- }- if (path == null) {- path = "";- }-- FileStore.FileType typ = fileStore.getType(path, false);- if (typ == FileStore.FileType.NOFILE) {- rsp.add("files", Collections.singletonMap(path, null));- return;- }- if (typ == FileStore.FileType.DIRECTORY) {- rsp.add("files", Collections.singletonMap(path, fileStore.list(path, null)));- return;- }- if (req.getParams().getBool("meta", false)) {- if (typ == FileStore.FileType.FILE) {- int idx = path.lastIndexOf('/');- String fileName = path.substring(idx + 1);- String parentPath = path.substring(0, path.lastIndexOf('/'));- List<FileStore.FileDetails> l = fileStore.list(parentPath, s -> s.equals(fileName));- rsp.add("files", Collections.singletonMap(path, l.isEmpty() ? null : l.get(0)));- return;- }- } else {- writeRawFile(req, rsp, path);- }- }-- private void writeRawFile(SolrQueryRequest req, SolrQueryResponse rsp, String path) {- ModifiableSolrParams solrParams = new ModifiableSolrParams();- if ("json".equals(req.getParams().get(CommonParams.WT))) {- solrParams.add(CommonParams.WT, "json");- req.setParams(SolrParams.wrapDefaults(solrParams, req.getParams()));- try {- fileStore.get(- path,- it -> {- try {- InputStream inputStream = it.getInputStream();- if (inputStream != null) {- rsp.addResponse(new String(inputStream.readAllBytes(), UTF_8));- }- } catch (IOException e) {- throw new SolrException(- SolrException.ErrorCode.SERVER_ERROR, "Error reading file " + path);- }- },- false);- } catch (IOException e) {- throw new SolrException(- SolrException.ErrorCode.SERVER_ERROR, "Error getting file from path " + path);- }- } else {- solrParams.add(CommonParams.WT, FILE_STREAM);- req.setParams(SolrParams.wrapDefaults(solrParams, req.getParams()));- rsp.add(- FILE_STREAM,- (SolrCore.RawWriter)- os ->- fileStore.get(- path,- it -> {- try {- InputStream inputStream = it.getInputStream();- if (inputStream != null) {- inputStream.transferTo(os);- }- } catch (IOException e) {- throw new SolrException(- SolrException.ErrorCode.SERVER_ERROR, "Error reading file " + path);- }- },- false));- }- }- }-public static class MetaData implements MapWriter {public static final String SHA512 = "sha512";String sha512;@@ -223,88 +66,4 @@ public class FileStoreAPI {return false;}}-- static final String INVALIDCHARS = " /\\#&*\n\t%@~`=+^$><?{}[]|:;!";-- public static void validateName(String path, boolean failForTrusted) {- if (path == null) {- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "empty path");- }- List<String> parts = StrUtils.splitSmart(path, '/', true);- for (String part : parts) {- if (part.charAt(0) == '.') {- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "cannot start with period");- }- for (int i = 0; i < part.length(); i++) {- for (int j = 0; j < INVALIDCHARS.length(); j++) {- if (part.charAt(i) == INVALIDCHARS.charAt(j))- throw new SolrException(- SolrException.ErrorCode.BAD_REQUEST, "Unsupported char in file name: " + part);- }- }- }- if (failForTrusted && TRUSTED_DIR.equals(parts.get(0))) {- throw new SolrException(- SolrException.ErrorCode.BAD_REQUEST, "trying to write into /_trusted_/ directory");- }- }-- /**- * Validate a file for signature- *- * @param sigs the signatures. atleast one should succeed- * @param entry The file details- * @param isFirstAttempt If there is a failure- */- public void validate(List<String> sigs, FileStore.FileEntry entry, boolean isFirstAttempt)- throws SolrException, IOException {- if (!isFirstAttempt) {- // we are retrying because last validation failed.- // get all keys again and try again- fileStore.refresh(KEYS_DIR);- }-- Map<String, byte[]> keys = fileStore.getKeys();- if (keys == null || keys.isEmpty()) {- if (isFirstAttempt) {- validate(sigs, entry, false);- return;- }- throw new SolrException(- SolrException.ErrorCode.BAD_REQUEST, "Filestore does not have any public keys");- }- CryptoKeys cryptoKeys = null;- try {- cryptoKeys = new CryptoKeys(keys);- } catch (Exception e) {- throw new SolrException(- SolrException.ErrorCode.SERVER_ERROR, "Error parsing public keys in ZooKeeper");- }- for (String sig : sigs) {- Supplier<String> errMsg =- () ->- "Signature does not match any public key : "- + sig- + "sha256 "- + entry.getMetaData().sha512;- if (entry.getBuffer() != null) {- if (cryptoKeys.verify(sig, entry.getBuffer()) == null) {- if (isFirstAttempt) {- validate(sigs, entry, false);- return;- }- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.get());- }- } else {- InputStream inputStream = entry.getInputStream();- if (cryptoKeys.verify(sig, inputStream) == null) {- if (isFirstAttempt) {- validate(sigs, entry, false);- return;- }- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.get());- }- }- }- }}diff --git a/solr/core/src/java/org/apache/solr/filestore/NodeFileStore.java b/solr/core/src/java/org/apache/solr/filestore/NodeFileStore.javanew file mode 100644index 00000000000..2b12274ad26--- /dev/null+++ b/solr/core/src/java/org/apache/solr/filestore/NodeFileStore.java@@ -0,0 +1,210 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.filestore;++import static java.nio.charset.StandardCharsets.UTF_8;+import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;+import static org.apache.solr.response.RawResponseWriter.CONTENT;+import static org.apache.solr.security.PermissionNameProvider.Name.FILESTORE_READ_PERM;++import jakarta.inject.Inject;+import java.io.IOException;+import java.io.InputStream;+import java.lang.invoke.MethodHandles;+import java.util.Collections;+import java.util.List;+import java.util.stream.Collectors;+import org.apache.solr.api.JerseyResource;+import org.apache.solr.client.api.endpoint.NodeFileStoreApis;+import org.apache.solr.client.api.model.FileStoreDirectoryListingResponse;+import org.apache.solr.client.api.model.FileStoreEntryMetadata;+import org.apache.solr.client.api.model.FileStoreJsonFileResponse;+import org.apache.solr.client.api.model.SolrJerseyResponse;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.params.CommonParams;+import org.apache.solr.common.params.ModifiableSolrParams;+import org.apache.solr.common.params.SolrParams;+import org.apache.solr.core.CoreContainer;+import org.apache.solr.core.SolrCore;+import org.apache.solr.jersey.PermissionName;+import org.apache.solr.request.SolrQueryRequest;+import org.apache.solr.response.SolrQueryResponse;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++/** Implementation for {@link NodeFileStoreApis} */+public class NodeFileStore extends JerseyResource implements NodeFileStoreApis {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ private final CoreContainer coreContainer;+ private final SolrQueryRequest req;+ private final SolrQueryResponse rsp;+ private final FileStore fileStore;++ @Inject+ public NodeFileStore(+ CoreContainer coreContainer,+ DistribFileStore fileStore,+ SolrQueryRequest req,+ SolrQueryResponse rsp) {+ this.coreContainer = coreContainer;+ this.req = req;+ this.rsp = rsp;+ this.fileStore = fileStore;+ }++ // TODO SOLR-17351 - this single "get" operation actually supports several different chunks of+ // functionality: syncing, directory listing, file-fetching, metadata-fetching. We should split+ // it up into multiple distinct APIs+ @Override+ @PermissionName(FILESTORE_READ_PERM)+ public SolrJerseyResponse getFile(String path, Boolean sync, String getFrom, Boolean meta) {+ final var response = instantiateJerseyResponse(SolrJerseyResponse.class);++ if (Boolean.TRUE.equals(sync)) {+ try {+ fileStore.syncToAllNodes(path);+ return response;+ } catch (IOException e) {+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error getting file ", e);+ }+ }++ if (path == null) {+ path = "";+ }+ final var pathCopy = path;+ if (getFrom != null) {+ coreContainer+ .getUpdateShardHandler()+ .getUpdateExecutor()+ .submit(+ () -> {+ log.debug("Downloading file {}", pathCopy);+ try {+ fileStore.fetch(pathCopy, getFrom);+ } catch (Exception e) {+ log.error("Failed to download file: {}", pathCopy, e);+ }+ log.info("downloaded file: {}", pathCopy);+ });+ return response;+ }++ FileStore.FileType type = fileStore.getType(path, false);+ if (type == FileStore.FileType.NOFILE) {+ final var fileMissingResponse =+ instantiateJerseyResponse(FileStoreDirectoryListingResponse.class);+ fileMissingResponse.files = Collections.singletonMap(path, null);+ return fileMissingResponse;+ }+ if (type == FileStore.FileType.DIRECTORY) {+ final var directoryListingResponse =+ instantiateJerseyResponse(FileStoreDirectoryListingResponse.class);+ final var directoryContents =+ fileStore.list(path, null).stream()+ .map(details -> convertToResponse(details))+ .collect(Collectors.toList());+ directoryListingResponse.files = Collections.singletonMap(path, directoryContents);+ return directoryListingResponse;+ }+ if (Boolean.TRUE.equals(meta)) {+ if (type == FileStore.FileType.FILE) {+ int idx = path.lastIndexOf('/');+ String fileName = path.substring(idx + 1);+ String parentPath = path.substring(0, path.lastIndexOf('/'));+ List<FileStore.FileDetails> l = fileStore.list(parentPath, s -> s.equals(fileName));++ final var fileMetaResponse =+ instantiateJerseyResponse(FileStoreDirectoryListingResponse.class);+ fileMetaResponse.files =+ Collections.singletonMap(path, l.isEmpty() ? null : convertToResponse(l.get(0)));+ return fileMetaResponse;+ }+ } else { // User wants to get the "raw" file+ // TODO Should we be trying to json-ify otherwise "raw" files in this way? It seems like a+ // pretty sketchy idea, esp. for code with very little test coverage. Consider removing+ if ("json".equals(req.getParams().get(CommonParams.WT))) {+ final var jsonResponse = instantiateJerseyResponse(FileStoreJsonFileResponse.class);+ try {+ fileStore.get(+ pathCopy,+ it -> {+ try {+ InputStream inputStream = it.getInputStream();+ if (inputStream != null) {+ jsonResponse.response = new String(inputStream.readAllBytes(), UTF_8);+ }+ } catch (IOException e) {+ throw new SolrException(+ SolrException.ErrorCode.SERVER_ERROR, "Error reading file " + pathCopy);+ }+ },+ false);+ return jsonResponse;+ } catch (IOException e) {+ throw new SolrException(+ SolrException.ErrorCode.SERVER_ERROR, "Error getting file from path " + path);+ }+ } else {+ ModifiableSolrParams solrParams = new ModifiableSolrParams();+ solrParams.add(CommonParams.WT, FILE_STREAM);+ req.setParams(SolrParams.wrapDefaults(solrParams, req.getParams()));+ rsp.add(+ CONTENT,+ (SolrCore.RawWriter)+ os ->+ fileStore.get(+ pathCopy,+ it -> {+ try {+ InputStream inputStream = it.getInputStream();+ if (inputStream != null) {+ inputStream.transferTo(os);+ }+ } catch (IOException e) {+ throw new SolrException(+ SolrException.ErrorCode.SERVER_ERROR,+ "Error reading file " + pathCopy);+ }+ },+ false));+ }+ }+ return response;+ }++ // TODO Modify the filestore implementation itself to return this object, so conversion isn't+ // needed.+ private FileStoreEntryMetadata convertToResponse(FileStore.FileDetails details) {+ final var entryMetadata = new FileStoreEntryMetadata();++ entryMetadata.name = details.getSimpleName();+ if (details.isDir()) {+ entryMetadata.dir = true;+ return entryMetadata;+ }++ entryMetadata.size = details.size();+ entryMetadata.timestamp = details.getTimeStamp();+ if (details.getMetaData() != null) {+ details.getMetaData().toMap(entryMetadata.unknownProperties());+ }++ return entryMetadata;+ }+}diff --git a/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.java b/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.javaindex c317dc8fac4..e2a907d07d4 100644--- a/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.java+++ b/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.java@@ -19,6 +19,7 @@ package org.apache.solr.handler.api;import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE_V2;import static org.apache.solr.common.params.CommonParams.WT;+import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;import com.fasterxml.jackson.annotation.JsonAnyGetter;import com.fasterxml.jackson.annotation.JsonProperty;@@ -30,6 +31,7 @@ import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.StrUtils;import org.apache.solr.common.util.Utils;+import org.apache.solr.response.RawResponseWriter;import org.apache.solr.response.SolrQueryResponse;/** Utilities helpful for common V2 API declaration tasks. */@@ -100,6 +102,8 @@ public class V2ApiUtils {return "application/xml";case "javabin":return BINARY_CONTENT_TYPE_V2;+ case FILE_STREAM:+ return RawResponseWriter.CONTENT_TYPE;default:return defaultMediaType;}diff --git a/solr/core/src/java/org/apache/solr/jersey/SolrRequestAuthorizer.java b/solr/core/src/java/org/apache/solr/jersey/SolrRequestAuthorizer.javaindex 06e62e4f463..3ff3baf5f70 100644--- a/solr/core/src/java/org/apache/solr/jersey/SolrRequestAuthorizer.java+++ b/solr/core/src/java/org/apache/solr/jersey/SolrRequestAuthorizer.java@@ -77,9 +77,11 @@ public class SolrRequestAuthorizer implements ContainerRequestFilter {** Since we don't invoke Jersey code in those particular cases we can ignore those checks here.*/- if (coreContainer.getAuthorizationPlugin() == null) {+ if (coreContainer.getAuthorizationPlugin() == null+ || isAlreadyAuthorizedByPKI(coreContainer, servletRequest)) {return;}+final AuthorizationContext authzContext =getAuthzContext(servletRequest, requestType, collectionNames, solrParams);log.debug("Attempting authz with context {}", authzContext);@@ -92,6 +94,18 @@ public class SolrRequestAuthorizer implements ContainerRequestFilter {}}+ private boolean isAlreadyAuthorizedByPKI(CoreContainer coreContainer, HttpServletRequest req) {+ if (coreContainer.getPkiAuthenticationSecurityBuilder() != null+ && req.getUserPrincipal() != null) {+ boolean needsAuthz =+ coreContainer.getPkiAuthenticationSecurityBuilder().needsAuthorization(req);+ log.debug("PkiAuthenticationPlugin says authorization required : {} ", needsAuthz);+ return !needsAuthz;+ }++ return false;+ }+private AuthorizationContext getAuthzContext(HttpServletRequest servletRequest,AuthorizationContext.RequestType reqType,diff --git a/solr/core/src/java/org/apache/solr/response/RawResponseWriter.java b/solr/core/src/java/org/apache/solr/response/RawResponseWriter.javaindex 20c257fc25f..34d7e112ed2 100644--- a/solr/core/src/java/org/apache/solr/response/RawResponseWriter.java+++ b/solr/core/src/java/org/apache/solr/response/RawResponseWriter.java@@ -16,6 +16,7 @@*/package org.apache.solr.response;+import java.io.Closeable;import java.io.IOException;import java.io.InputStream;import java.io.OutputStream;@@ -23,6 +24,7 @@ import java.io.Reader;import java.io.Writer;import org.apache.solr.common.util.ContentStream;import org.apache.solr.common.util.NamedList;+import org.apache.solr.core.SolrCore;import org.apache.solr.request.SolrQueryRequest;/**@@ -117,6 +119,10 @@ public class RawResponseWriter implements BinaryQueryResponseWriter {try (InputStream in = content.getStream()) {in.transferTo(out);}+ } else if (obj != null && (obj instanceof SolrCore.RawWriter)) {+ final var rawWriter = (SolrCore.RawWriter) obj;+ rawWriter.write(out);+ if (rawWriter instanceof Closeable) ((Closeable) rawWriter).close();} else {QueryResponseWriterUtil.writeQueryResponse(out, getBaseWriter(request), request, response, getContentType(request, response));diff --git a/solr/core/src/test/org/apache/solr/handler/api/V2ApiUtilsTest.java b/solr/core/src/test/org/apache/solr/handler/api/V2ApiUtilsTest.javaindex 38cbf00a2fa..51b89243dbe 100644--- a/solr/core/src/test/org/apache/solr/handler/api/V2ApiUtilsTest.java+++ b/solr/core/src/test/org/apache/solr/handler/api/V2ApiUtilsTest.java@@ -17,6 +17,7 @@package org.apache.solr.handler.api;import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE_V2;+import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;import jakarta.ws.rs.core.MediaType;import org.apache.solr.SolrTestCaseJ4;@@ -57,6 +58,10 @@ public class V2ApiUtilsTest extends SolrTestCaseJ4 {params.set("wt", "javabin");assertEquals(BINARY_CONTENT_TYPE_V2, V2ApiUtils.getMediaTypeFromWtParam(params, null));+ params.set("wt", FILE_STREAM);+ assertEquals(+ "application/vnd.apache.solr.raw", V2ApiUtils.getMediaTypeFromWtParam(params, null));+// Defaults to, well the default, whenever an unknown/unexpected/typo'd 'wt' is providedparams.set("wt", "josn");assertEquals("someDefault", V2ApiUtils.getMediaTypeFromWtParam(params, "someDefault"));diff --git a/solr/solrj/src/resources/java-template/api.mustache b/solr/solrj/src/resources/java-template/api.mustacheindex a4171667b0a..bbb300023ed 100644--- a/solr/solrj/src/resources/java-template/api.mustache+++ b/solr/solrj/src/resources/java-template/api.mustache@@ -73,7 +73,8 @@ import {{modelPackage}}.{{dataType}};public class {{classname}} {{{#operation}}- public static class {{operationIdCamelCase}}Response extends JacksonParsingResponse<{{modelPackage}}.{{returnType}}> {+ {{^vendorExtensions.x-omitFromCodegen}}+ public static class {{operationIdCamelCase}}Response extends JacksonParsingResponse<{{modelPackage}}.{{returnType}}> {public {{operationIdCamelCase}}Response() {super({{modelPackage}}.{{returnType}}.class);}@@ -206,6 +207,7 @@ public class {{classname}} {return new InputStreamResponseParser("json");}}+ {{/vendorExtensions.x-omitFromCodegen}}{{/operation}}}{{/operations}}