Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-17302: Convert remaining filestore APIs to JAX-RS (#2532)

Jason Gerlowskid0aad69c0ffc
diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java
index 07746b5e92d..11ee4e1e26d 100644
--- a/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java
+++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterFileStoreApis.java
@@ -64,5 +64,10 @@ public interface ClusterFileStoreApis {
SolrJerseyResponse deleteFile(
@Parameter(description = "Path to a file or directory within the filestore")
@PathParam("path")
- String path);
+ String path,
+ @Parameter(
+ description =
+ "Indicates whether the deletion should only be done on the receiving node. For internal use only")
+ @QueryParam("localDelete")
+ Boolean localDelete);
}
diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeFileStoreApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeFileStoreApis.java
new file mode 100644
index 00000000000..15f4a73cfb1
--- /dev/null
+++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeFileStoreApis.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.api.endpoint;
+
+import static org.apache.solr.client.api.util.Constants.OMIT_FROM_CODEGEN_PROPERTY;
+
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.extensions.Extension;
+import io.swagger.v3.oas.annotations.extensions.ExtensionProperty;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.QueryParam;
+import org.apache.solr.client.api.model.SolrJerseyResponse;
+
+/**
+ * V2 APIs for fetching filestore files, syncing them across nodes, or fetching related metadata.
+ */
+@Path("/node")
+public interface NodeFileStoreApis {
+ @GET
+ @Operation(
+ summary = "Retrieve file contents or metadata from the filestore.",
+ tags = {"file-store"},
+ // The response of this v2 API is highly variable based on the parameters specified. It can
+ // return raw (potentially binary) file data, a JSON-ified representation of that file data,
+ // metadata regarding one or multiple file store entries, etc. This variability can be
+ // handled on the Jersey server side, but would be prohibitively difficult to accommodate in
+ // our code-generation templates. Ideally, cosmetic improvements (e.g. splitting it up into
+ // multiple endpoints) will make this unnecessary in the future. But for now, the extension
+ // property below ensures that this endpoint is ignored entirely when doing code generation.
+ extensions = {
+ @Extension(
+ properties = {@ExtensionProperty(name = OMIT_FROM_CODEGEN_PROPERTY, value = "true")})
+ })
+ @Path("/files{path:.+}")
+ SolrJerseyResponse getFile(
+ @Parameter(description = "Path to a file or directory within the filestore")
+ @PathParam("path")
+ String path,
+ @Parameter(
+ description =
+ "If true, triggers syncing for this file across all nodes in the filestore")
+ @QueryParam("sync")
+ Boolean sync,
+ @Parameter(description = "An optional Solr node name to fetch the file from")
+ @QueryParam("getFrom")
+ String getFrom,
+ @Parameter(description = "Indicates that (only) file metadata should be fetched")
+ @QueryParam("meta")
+ Boolean meta);
+}
diff --git a/solr/api/src/java/org/apache/solr/client/api/model/FileStoreDirectoryListingResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/FileStoreDirectoryListingResponse.java
new file mode 100644
index 00000000000..bcbc5f1f728
--- /dev/null
+++ b/solr/api/src/java/org/apache/solr/client/api/model/FileStoreDirectoryListingResponse.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.api.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.Map;
+
+/**
+ * One of several possible responses from {@link
+ * org.apache.solr.client.api.endpoint.NodeFileStoreApis#getFile(String, Boolean, String, Boolean)}
+ */
+public class FileStoreDirectoryListingResponse extends SolrJerseyResponse {
+ @JsonProperty public Map<String, Object> files;
+}
diff --git a/solr/api/src/java/org/apache/solr/client/api/model/FileStoreEntryMetadata.java b/solr/api/src/java/org/apache/solr/client/api/model/FileStoreEntryMetadata.java
new file mode 100644
index 00000000000..5a9bc213452
--- /dev/null
+++ b/solr/api/src/java/org/apache/solr/client/api/model/FileStoreEntryMetadata.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.api.model;
+
+import com.fasterxml.jackson.annotation.JsonAnyGetter;
+import com.fasterxml.jackson.annotation.JsonAnySetter;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+/** Represents the metadata for a single filestore file or directory */
+public class FileStoreEntryMetadata {
+ @JsonProperty public String name;
+ @JsonProperty public Boolean dir;
+ @JsonProperty public Long size;
+ @JsonProperty public Date timestamp;
+
+ private Map<String, Object> additionalMetadata = new HashMap<>();
+
+ @JsonAnyGetter
+ public Map<String, Object> unknownProperties() {
+ return additionalMetadata;
+ }
+
+ @JsonAnySetter
+ public void setUnknownProperty(String field, Object value) {
+ additionalMetadata.put(field, value);
+ }
+}
diff --git a/solr/api/src/java/org/apache/solr/client/api/model/FileStoreJsonFileResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/FileStoreJsonFileResponse.java
new file mode 100644
index 00000000000..321faec01ec
--- /dev/null
+++ b/solr/api/src/java/org/apache/solr/client/api/model/FileStoreJsonFileResponse.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.api.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * One of several possible responses from {@link
+ * org.apache.solr.client.api.endpoint.NodeFileStoreApis#getFile(String, Boolean, String, Boolean)}
+ *
+ * <p>Typically used when 'wt=json' is specified while retrieving an individual file from the
+ * filestore
+ */
+public class FileStoreJsonFileResponse extends SolrJerseyResponse {
+ @JsonProperty public String response;
+}
diff --git a/solr/api/src/java/org/apache/solr/client/api/util/Constants.java b/solr/api/src/java/org/apache/solr/client/api/util/Constants.java
index 560ce77af97..b4ef56c2050 100644
--- a/solr/api/src/java/org/apache/solr/client/api/util/Constants.java
+++ b/solr/api/src/java/org/apache/solr/client/api/util/Constants.java
@@ -27,6 +27,7 @@ public class Constants {
public static final String INDEX_PATH_PREFIX =
"/{" + INDEX_TYPE_PATH_PARAMETER + ":cores|collections}/{" + INDEX_NAME_PATH_PARAMETER + "}";
+ public static final String OMIT_FROM_CODEGEN_PROPERTY = "omitFromCodegen";
public static final String GENERIC_ENTITY_PROPERTY = "genericEntity";
public static final String BINARY_CONTENT_TYPE_V2 = "application/vnd.apache.solr.javabin";
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index a05bfed4e7d..f14680d5561 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -107,7 +107,7 @@ import org.apache.solr.core.backup.repository.BackupRepositoryFactory;
import org.apache.solr.filestore.ClusterFileStore;
import org.apache.solr.filestore.DistribFileStore;
import org.apache.solr.filestore.FileStore;
-import org.apache.solr.filestore.FileStoreAPI;
+import org.apache.solr.filestore.NodeFileStore;
import org.apache.solr.handler.ClusterAPI;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.handler.SnapShooter;
@@ -298,7 +298,6 @@ public class CoreContainer {
private DelegatingPlacementPluginFactory placementPluginFactory;
private DistribFileStore fileStore;
- private FileStoreAPI fileStoreAPI;
private ClusterFileStore clusterFileStoreAPI;
private SolrPackageLoader packageLoader;
@@ -862,10 +861,8 @@ public class CoreContainer {
pkiAuthenticationSecurityBuilder.initializeMetrics(solrMetricsContext, "/authentication/pki");
fileStore = new DistribFileStore(this);
- fileStoreAPI = new FileStoreAPI(this);
- registerV2ApiIfEnabled(fileStoreAPI.readAPI);
- registerV2ApiIfEnabled(fileStoreAPI.writeAPI);
registerV2ApiIfEnabled(ClusterFileStore.class);
+ registerV2ApiIfEnabled(NodeFileStore.class);
packageLoader = new SolrPackageLoader(this);
registerV2ApiIfEnabled(packageLoader.getPackageAPI().editAPI);
diff --git a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java
index a664e5699cb..db422baf6bf 100644
--- a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java
+++ b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java
@@ -141,12 +141,15 @@ public class ClusterFileStore extends JerseyResource implements ClusterFileStore
return response;
}
- @Override
- @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
- public SolrJerseyResponse deleteFile(String filePath) {
- final var response = instantiateJerseyResponse(SolrJerseyResponse.class);
- if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) {
- throw new RuntimeException(PackageAPI.ERR_MSG);
+ private void doLocalDelete(String filePath) {
+ fileStore.deleteLocal(filePath);
+ }
+
+ private void doClusterDelete(String filePath) {
+ FileStore.FileType type = fileStore.getType(filePath, true);
+ if (type == FileStore.FileType.NOFILE) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST, "Path does not exist: " + filePath);
}
try {
@@ -154,19 +157,7 @@ public class ClusterFileStore extends JerseyResource implements ClusterFileStore
.getZkController()
.getZkClient()
.create(TMP_ZK_NODE, "true".getBytes(UTF_8), CreateMode.EPHEMERAL, true);
- validateName(filePath, true);
- if (coreContainer.getPackageLoader().getPackageAPI().isJarInuse(filePath)) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "jar in use, can't delete");
- }
- FileStore.FileType type = fileStore.getType(filePath, true);
- if (type == FileStore.FileType.NOFILE) {
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST, "Path does not exist: " + filePath);
- }
fileStore.delete(filePath);
- return response;
- } catch (SolrException e) {
- throw e;
} catch (Exception e) {
log.error("Unknown error", e);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
@@ -179,6 +170,30 @@ public class ClusterFileStore extends JerseyResource implements ClusterFileStore
}
}
+ private void doDelete(String filePath, Boolean localDelete) {
+ if (Boolean.TRUE.equals(localDelete)) {
+ doLocalDelete(filePath);
+ } else {
+ doClusterDelete(filePath);
+ }
+ }
+
+ @Override
+ @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
+ public SolrJerseyResponse deleteFile(String filePath, Boolean localDelete) {
+ final var response = instantiateJerseyResponse(SolrJerseyResponse.class);
+ if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) {
+ throw new RuntimeException(PackageAPI.ERR_MSG);
+ }
+
+ validateName(filePath, true);
+ if (coreContainer.getPackageLoader().getPackageAPI().isJarInuse(filePath)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "jar in use, can't delete");
+ }
+ doDelete(filePath, localDelete);
+ return response;
+ }
+
private List<String> readSignatures(List<String> signatures, byte[] buf)
throws SolrException, IOException {
if (signatures == null || signatures.isEmpty()) return null;
diff --git a/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java b/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java
index 7eb73354e6e..1a462dee26e 100644
--- a/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java
+++ b/solr/core/src/java/org/apache/solr/filestore/DistribFileStore.java
@@ -50,7 +50,6 @@ import org.apache.http.client.methods.HttpDelete;
import org.apache.lucene.util.IOUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrPaths;
@@ -288,20 +287,6 @@ public class DistribFileStore implements FileStore {
return realPath().toFile().length();
}
- @Override
- public void writeMap(EntryWriter ew) throws IOException {
- MetaData metaData = readMetaData();
- ew.put(CommonParams.NAME, getSimpleName());
- if (type == FileType.DIRECTORY) {
- ew.put("dir", true);
- return;
- }
-
- ew.put("size", size());
- ew.put("timestamp", getTimeStamp());
- if (metaData != null) metaData.writeMap(ew);
- }
-
@Override
public String getSimpleName() {
int idx = path.lastIndexOf('/');
@@ -503,7 +488,7 @@ public class DistribFileStore implements FileStore {
for (String node : nodes) {
String baseUrl =
coreContainer.getZkController().getZkStateReader().getBaseUrlV2ForNodeName(node);
- String url = baseUrl + "/node/files" + path;
+ String url = baseUrl + "/cluster/files" + path + "?localDelete=true";
HttpDelete del = new HttpDelete(url);
// invoke delete command on all nodes asynchronously
coreContainer.runAsync(() -> Utils.executeHttpMethod(client, url, null, del));
diff --git a/solr/core/src/java/org/apache/solr/filestore/FileStore.java b/solr/core/src/java/org/apache/solr/filestore/FileStore.java
index 65673757438..916b8b33b3b 100644
--- a/solr/core/src/java/org/apache/solr/filestore/FileStore.java
+++ b/solr/core/src/java/org/apache/solr/filestore/FileStore.java
@@ -26,7 +26,6 @@ import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Predicate;
-import org.apache.solr.common.MapWriter;
import org.apache.solr.filestore.FileStoreAPI.MetaData;
import org.apache.zookeeper.server.ByteBufferInputStream;
@@ -109,7 +108,7 @@ public interface FileStore {
METADATA
}
- interface FileDetails extends MapWriter {
+ interface FileDetails {
String getSimpleName();
diff --git a/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java b/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java
index d54acc464f5..14e8e1e0c34 100644
--- a/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java
+++ b/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java
@@ -17,173 +17,16 @@
package org.apache.solr.filestore;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;
-
import java.io.IOException;
-import java.io.InputStream;
-import java.lang.invoke.MethodHandles;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import java.util.function.Supplier;
-import org.apache.solr.api.EndPoint;
-import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.common.MapWriter;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.CoreContainer;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.security.PermissionNameProvider;
-import org.apache.solr.util.CryptoKeys;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
public class FileStoreAPI {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- public static final String FILESTORE_DIRECTORY = "filestore";
- public static final String TRUSTED_DIR = "_trusted_";
public static final String KEYS_DIR = "/_trusted_/keys";
- private final CoreContainer coreContainer;
- FileStore fileStore;
- public final FSRead readAPI = new FSRead();
- public final FSWrite writeAPI = new FSWrite();
-
- public FileStoreAPI(CoreContainer coreContainer) {
- this.coreContainer = coreContainer;
- fileStore = coreContainer.getFileStore();
- }
-
- public class FSWrite {
-
- @EndPoint(
- path = "/node/files/*",
- method = SolrRequest.METHOD.DELETE,
- permission = PermissionNameProvider.Name.FILESTORE_WRITE_PERM)
- public void deleteLocal(SolrQueryRequest req, SolrQueryResponse rsp) {
- String path = req.getPathTemplateValues().get("*");
- validateName(path, true);
- fileStore.deleteLocal(path);
- }
- }
-
- public class FSRead {
- @EndPoint(
- path = "/node/files/*",
- method = SolrRequest.METHOD.GET,
- permission = PermissionNameProvider.Name.FILESTORE_READ_PERM)
- public void read(SolrQueryRequest req, SolrQueryResponse rsp) {
- String path = req.getPathTemplateValues().get("*");
- String pathCopy = path;
- if (req.getParams().getBool("sync", false)) {
- try {
- fileStore.syncToAllNodes(path);
- return;
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error getting file ", e);
- }
- }
- String getFrom = req.getParams().get("getFrom");
- if (getFrom != null) {
- coreContainer
- .getUpdateShardHandler()
- .getUpdateExecutor()
- .submit(
- () -> {
- log.debug("Downloading file {}", pathCopy);
- try {
- fileStore.fetch(pathCopy, getFrom);
- } catch (Exception e) {
- log.error("Failed to download file: {}", pathCopy, e);
- }
- log.info("downloaded file: {}", pathCopy);
- });
- return;
- }
- if (path == null) {
- path = "";
- }
-
- FileStore.FileType typ = fileStore.getType(path, false);
- if (typ == FileStore.FileType.NOFILE) {
- rsp.add("files", Collections.singletonMap(path, null));
- return;
- }
- if (typ == FileStore.FileType.DIRECTORY) {
- rsp.add("files", Collections.singletonMap(path, fileStore.list(path, null)));
- return;
- }
- if (req.getParams().getBool("meta", false)) {
- if (typ == FileStore.FileType.FILE) {
- int idx = path.lastIndexOf('/');
- String fileName = path.substring(idx + 1);
- String parentPath = path.substring(0, path.lastIndexOf('/'));
- List<FileStore.FileDetails> l = fileStore.list(parentPath, s -> s.equals(fileName));
- rsp.add("files", Collections.singletonMap(path, l.isEmpty() ? null : l.get(0)));
- return;
- }
- } else {
- writeRawFile(req, rsp, path);
- }
- }
-
- private void writeRawFile(SolrQueryRequest req, SolrQueryResponse rsp, String path) {
- ModifiableSolrParams solrParams = new ModifiableSolrParams();
- if ("json".equals(req.getParams().get(CommonParams.WT))) {
- solrParams.add(CommonParams.WT, "json");
- req.setParams(SolrParams.wrapDefaults(solrParams, req.getParams()));
- try {
- fileStore.get(
- path,
- it -> {
- try {
- InputStream inputStream = it.getInputStream();
- if (inputStream != null) {
- rsp.addResponse(new String(inputStream.readAllBytes(), UTF_8));
- }
- } catch (IOException e) {
- throw new SolrException(
- SolrException.ErrorCode.SERVER_ERROR, "Error reading file " + path);
- }
- },
- false);
- } catch (IOException e) {
- throw new SolrException(
- SolrException.ErrorCode.SERVER_ERROR, "Error getting file from path " + path);
- }
- } else {
- solrParams.add(CommonParams.WT, FILE_STREAM);
- req.setParams(SolrParams.wrapDefaults(solrParams, req.getParams()));
- rsp.add(
- FILE_STREAM,
- (SolrCore.RawWriter)
- os ->
- fileStore.get(
- path,
- it -> {
- try {
- InputStream inputStream = it.getInputStream();
- if (inputStream != null) {
- inputStream.transferTo(os);
- }
- } catch (IOException e) {
- throw new SolrException(
- SolrException.ErrorCode.SERVER_ERROR, "Error reading file " + path);
- }
- },
- false));
- }
- }
- }
-
public static class MetaData implements MapWriter {
public static final String SHA512 = "sha512";
String sha512;
@@ -223,88 +66,4 @@ public class FileStoreAPI {
return false;
}
}
-
- static final String INVALIDCHARS = " /\\#&*\n\t%@~`=+^$><?{}[]|:;!";
-
- public static void validateName(String path, boolean failForTrusted) {
- if (path == null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "empty path");
- }
- List<String> parts = StrUtils.splitSmart(path, '/', true);
- for (String part : parts) {
- if (part.charAt(0) == '.') {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "cannot start with period");
- }
- for (int i = 0; i < part.length(); i++) {
- for (int j = 0; j < INVALIDCHARS.length(); j++) {
- if (part.charAt(i) == INVALIDCHARS.charAt(j))
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST, "Unsupported char in file name: " + part);
- }
- }
- }
- if (failForTrusted && TRUSTED_DIR.equals(parts.get(0))) {
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST, "trying to write into /_trusted_/ directory");
- }
- }
-
- /**
- * Validate a file for signature
- *
- * @param sigs the signatures. atleast one should succeed
- * @param entry The file details
- * @param isFirstAttempt If there is a failure
- */
- public void validate(List<String> sigs, FileStore.FileEntry entry, boolean isFirstAttempt)
- throws SolrException, IOException {
- if (!isFirstAttempt) {
- // we are retrying because last validation failed.
- // get all keys again and try again
- fileStore.refresh(KEYS_DIR);
- }
-
- Map<String, byte[]> keys = fileStore.getKeys();
- if (keys == null || keys.isEmpty()) {
- if (isFirstAttempt) {
- validate(sigs, entry, false);
- return;
- }
- throw new SolrException(
- SolrException.ErrorCode.BAD_REQUEST, "Filestore does not have any public keys");
- }
- CryptoKeys cryptoKeys = null;
- try {
- cryptoKeys = new CryptoKeys(keys);
- } catch (Exception e) {
- throw new SolrException(
- SolrException.ErrorCode.SERVER_ERROR, "Error parsing public keys in ZooKeeper");
- }
- for (String sig : sigs) {
- Supplier<String> errMsg =
- () ->
- "Signature does not match any public key : "
- + sig
- + "sha256 "
- + entry.getMetaData().sha512;
- if (entry.getBuffer() != null) {
- if (cryptoKeys.verify(sig, entry.getBuffer()) == null) {
- if (isFirstAttempt) {
- validate(sigs, entry, false);
- return;
- }
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.get());
- }
- } else {
- InputStream inputStream = entry.getInputStream();
- if (cryptoKeys.verify(sig, inputStream) == null) {
- if (isFirstAttempt) {
- validate(sigs, entry, false);
- return;
- }
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.get());
- }
- }
- }
- }
}
diff --git a/solr/core/src/java/org/apache/solr/filestore/NodeFileStore.java b/solr/core/src/java/org/apache/solr/filestore/NodeFileStore.java
new file mode 100644
index 00000000000..2b12274ad26
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/filestore/NodeFileStore.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.filestore;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;
+import static org.apache.solr.response.RawResponseWriter.CONTENT;
+import static org.apache.solr.security.PermissionNameProvider.Name.FILESTORE_READ_PERM;
+
+import jakarta.inject.Inject;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.invoke.MethodHandles;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.solr.api.JerseyResource;
+import org.apache.solr.client.api.endpoint.NodeFileStoreApis;
+import org.apache.solr.client.api.model.FileStoreDirectoryListingResponse;
+import org.apache.solr.client.api.model.FileStoreEntryMetadata;
+import org.apache.solr.client.api.model.FileStoreJsonFileResponse;
+import org.apache.solr.client.api.model.SolrJerseyResponse;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.jersey.PermissionName;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Implementation for {@link NodeFileStoreApis} */
+public class NodeFileStore extends JerseyResource implements NodeFileStoreApis {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final CoreContainer coreContainer;
+ private final SolrQueryRequest req;
+ private final SolrQueryResponse rsp;
+ private final FileStore fileStore;
+
+ @Inject
+ public NodeFileStore(
+ CoreContainer coreContainer,
+ DistribFileStore fileStore,
+ SolrQueryRequest req,
+ SolrQueryResponse rsp) {
+ this.coreContainer = coreContainer;
+ this.req = req;
+ this.rsp = rsp;
+ this.fileStore = fileStore;
+ }
+
+ // TODO SOLR-17351 - this single "get" operation actually supports several different chunks of
+ // functionality: syncing, directory listing, file-fetching, metadata-fetching. We should split
+ // it up into multiple distinct APIs
+ @Override
+ @PermissionName(FILESTORE_READ_PERM)
+ public SolrJerseyResponse getFile(String path, Boolean sync, String getFrom, Boolean meta) {
+ final var response = instantiateJerseyResponse(SolrJerseyResponse.class);
+
+ if (Boolean.TRUE.equals(sync)) {
+ try {
+ fileStore.syncToAllNodes(path);
+ return response;
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error getting file ", e);
+ }
+ }
+
+ if (path == null) {
+ path = "";
+ }
+ final var pathCopy = path;
+ if (getFrom != null) {
+ coreContainer
+ .getUpdateShardHandler()
+ .getUpdateExecutor()
+ .submit(
+ () -> {
+ log.debug("Downloading file {}", pathCopy);
+ try {
+ fileStore.fetch(pathCopy, getFrom);
+ } catch (Exception e) {
+ log.error("Failed to download file: {}", pathCopy, e);
+ }
+ log.info("downloaded file: {}", pathCopy);
+ });
+ return response;
+ }
+
+ FileStore.FileType type = fileStore.getType(path, false);
+ if (type == FileStore.FileType.NOFILE) {
+ final var fileMissingResponse =
+ instantiateJerseyResponse(FileStoreDirectoryListingResponse.class);
+ fileMissingResponse.files = Collections.singletonMap(path, null);
+ return fileMissingResponse;
+ }
+ if (type == FileStore.FileType.DIRECTORY) {
+ final var directoryListingResponse =
+ instantiateJerseyResponse(FileStoreDirectoryListingResponse.class);
+ final var directoryContents =
+ fileStore.list(path, null).stream()
+ .map(details -> convertToResponse(details))
+ .collect(Collectors.toList());
+ directoryListingResponse.files = Collections.singletonMap(path, directoryContents);
+ return directoryListingResponse;
+ }
+ if (Boolean.TRUE.equals(meta)) {
+ if (type == FileStore.FileType.FILE) {
+ int idx = path.lastIndexOf('/');
+ String fileName = path.substring(idx + 1);
+ String parentPath = path.substring(0, path.lastIndexOf('/'));
+ List<FileStore.FileDetails> l = fileStore.list(parentPath, s -> s.equals(fileName));
+
+ final var fileMetaResponse =
+ instantiateJerseyResponse(FileStoreDirectoryListingResponse.class);
+ fileMetaResponse.files =
+ Collections.singletonMap(path, l.isEmpty() ? null : convertToResponse(l.get(0)));
+ return fileMetaResponse;
+ }
+ } else { // User wants to get the "raw" file
+ // TODO Should we be trying to json-ify otherwise "raw" files in this way? It seems like a
+ // pretty sketchy idea, esp. for code with very little test coverage. Consider removing
+ if ("json".equals(req.getParams().get(CommonParams.WT))) {
+ final var jsonResponse = instantiateJerseyResponse(FileStoreJsonFileResponse.class);
+ try {
+ fileStore.get(
+ pathCopy,
+ it -> {
+ try {
+ InputStream inputStream = it.getInputStream();
+ if (inputStream != null) {
+ jsonResponse.response = new String(inputStream.readAllBytes(), UTF_8);
+ }
+ } catch (IOException e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR, "Error reading file " + pathCopy);
+ }
+ },
+ false);
+ return jsonResponse;
+ } catch (IOException e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR, "Error getting file from path " + path);
+ }
+ } else {
+ ModifiableSolrParams solrParams = new ModifiableSolrParams();
+ solrParams.add(CommonParams.WT, FILE_STREAM);
+ req.setParams(SolrParams.wrapDefaults(solrParams, req.getParams()));
+ rsp.add(
+ CONTENT,
+ (SolrCore.RawWriter)
+ os ->
+ fileStore.get(
+ pathCopy,
+ it -> {
+ try {
+ InputStream inputStream = it.getInputStream();
+ if (inputStream != null) {
+ inputStream.transferTo(os);
+ }
+ } catch (IOException e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Error reading file " + pathCopy);
+ }
+ },
+ false));
+ }
+ }
+ return response;
+ }
+
+ // TODO Modify the filestore implementation itself to return this object, so conversion isn't
+ // needed.
+ private FileStoreEntryMetadata convertToResponse(FileStore.FileDetails details) {
+ final var entryMetadata = new FileStoreEntryMetadata();
+
+ entryMetadata.name = details.getSimpleName();
+ if (details.isDir()) {
+ entryMetadata.dir = true;
+ return entryMetadata;
+ }
+
+ entryMetadata.size = details.size();
+ entryMetadata.timestamp = details.getTimeStamp();
+ if (details.getMetaData() != null) {
+ details.getMetaData().toMap(entryMetadata.unknownProperties());
+ }
+
+ return entryMetadata;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.java b/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.java
index c317dc8fac4..e2a907d07d4 100644
--- a/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.java
+++ b/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.java
@@ -19,6 +19,7 @@ package org.apache.solr.handler.api;
import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE_V2;
import static org.apache.solr.common.params.CommonParams.WT;
+import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;
import com.fasterxml.jackson.annotation.JsonAnyGetter;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -30,6 +31,7 @@ import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
+import org.apache.solr.response.RawResponseWriter;
import org.apache.solr.response.SolrQueryResponse;
/** Utilities helpful for common V2 API declaration tasks. */
@@ -100,6 +102,8 @@ public class V2ApiUtils {
return "application/xml";
case "javabin":
return BINARY_CONTENT_TYPE_V2;
+ case FILE_STREAM:
+ return RawResponseWriter.CONTENT_TYPE;
default:
return defaultMediaType;
}
diff --git a/solr/core/src/java/org/apache/solr/jersey/SolrRequestAuthorizer.java b/solr/core/src/java/org/apache/solr/jersey/SolrRequestAuthorizer.java
index 06e62e4f463..3ff3baf5f70 100644
--- a/solr/core/src/java/org/apache/solr/jersey/SolrRequestAuthorizer.java
+++ b/solr/core/src/java/org/apache/solr/jersey/SolrRequestAuthorizer.java
@@ -77,9 +77,11 @@ public class SolrRequestAuthorizer implements ContainerRequestFilter {
*
* Since we don't invoke Jersey code in those particular cases we can ignore those checks here.
*/
- if (coreContainer.getAuthorizationPlugin() == null) {
+ if (coreContainer.getAuthorizationPlugin() == null
+ || isAlreadyAuthorizedByPKI(coreContainer, servletRequest)) {
return;
}
+
final AuthorizationContext authzContext =
getAuthzContext(servletRequest, requestType, collectionNames, solrParams);
log.debug("Attempting authz with context {}", authzContext);
@@ -92,6 +94,18 @@ public class SolrRequestAuthorizer implements ContainerRequestFilter {
}
}
+ private boolean isAlreadyAuthorizedByPKI(CoreContainer coreContainer, HttpServletRequest req) {
+ if (coreContainer.getPkiAuthenticationSecurityBuilder() != null
+ && req.getUserPrincipal() != null) {
+ boolean needsAuthz =
+ coreContainer.getPkiAuthenticationSecurityBuilder().needsAuthorization(req);
+ log.debug("PkiAuthenticationPlugin says authorization required : {} ", needsAuthz);
+ return !needsAuthz;
+ }
+
+ return false;
+ }
+
private AuthorizationContext getAuthzContext(
HttpServletRequest servletRequest,
AuthorizationContext.RequestType reqType,
diff --git a/solr/core/src/java/org/apache/solr/response/RawResponseWriter.java b/solr/core/src/java/org/apache/solr/response/RawResponseWriter.java
index 20c257fc25f..34d7e112ed2 100644
--- a/solr/core/src/java/org/apache/solr/response/RawResponseWriter.java
+++ b/solr/core/src/java/org/apache/solr/response/RawResponseWriter.java
@@ -16,6 +16,7 @@
*/
package org.apache.solr.response;
+import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -23,6 +24,7 @@ import java.io.Reader;
import java.io.Writer;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
/**
@@ -117,6 +119,10 @@ public class RawResponseWriter implements BinaryQueryResponseWriter {
try (InputStream in = content.getStream()) {
in.transferTo(out);
}
+ } else if (obj != null && (obj instanceof SolrCore.RawWriter)) {
+ final var rawWriter = (SolrCore.RawWriter) obj;
+ rawWriter.write(out);
+ if (rawWriter instanceof Closeable) ((Closeable) rawWriter).close();
} else {
QueryResponseWriterUtil.writeQueryResponse(
out, getBaseWriter(request), request, response, getContentType(request, response));
diff --git a/solr/core/src/test/org/apache/solr/handler/api/V2ApiUtilsTest.java b/solr/core/src/test/org/apache/solr/handler/api/V2ApiUtilsTest.java
index 38cbf00a2fa..51b89243dbe 100644
--- a/solr/core/src/test/org/apache/solr/handler/api/V2ApiUtilsTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/api/V2ApiUtilsTest.java
@@ -17,6 +17,7 @@
package org.apache.solr.handler.api;
import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE_V2;
+import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;
import jakarta.ws.rs.core.MediaType;
import org.apache.solr.SolrTestCaseJ4;
@@ -57,6 +58,10 @@ public class V2ApiUtilsTest extends SolrTestCaseJ4 {
params.set("wt", "javabin");
assertEquals(BINARY_CONTENT_TYPE_V2, V2ApiUtils.getMediaTypeFromWtParam(params, null));
+ params.set("wt", FILE_STREAM);
+ assertEquals(
+ "application/vnd.apache.solr.raw", V2ApiUtils.getMediaTypeFromWtParam(params, null));
+
// Defaults to, well the default, whenever an unknown/unexpected/typo'd 'wt' is provided
params.set("wt", "josn");
assertEquals("someDefault", V2ApiUtils.getMediaTypeFromWtParam(params, "someDefault"));
diff --git a/solr/solrj/src/resources/java-template/api.mustache b/solr/solrj/src/resources/java-template/api.mustache
index a4171667b0a..bbb300023ed 100644
--- a/solr/solrj/src/resources/java-template/api.mustache
+++ b/solr/solrj/src/resources/java-template/api.mustache
@@ -73,7 +73,8 @@ import {{modelPackage}}.{{dataType}};
public class {{classname}} {
{{#operation}}
- public static class {{operationIdCamelCase}}Response extends JacksonParsingResponse<{{modelPackage}}.{{returnType}}> {
+ {{^vendorExtensions.x-omitFromCodegen}}
+ public static class {{operationIdCamelCase}}Response extends JacksonParsingResponse<{{modelPackage}}.{{returnType}}> {
public {{operationIdCamelCase}}Response() {
super({{modelPackage}}.{{returnType}}.class);
}
@@ -206,6 +207,7 @@ public class {{classname}} {
return new InputStreamResponseParser("json");
}
}
+ {{/vendorExtensions.x-omitFromCodegen}}
{{/operation}}
}
{{/operations}}