Apache Solr open-source search software
SOLR-15787: FileSystemConfigSetService: implement fully (#146)
Nazerke Seidana7143d4c8c16
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 44ece16aa8f..3c2b73e1a8d 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -85,6 +85,9 @@ Improvements* SOLR-8975: Introduce Builder setters for setters that exist on SolrClients for responseParser, requestWriter.Deprecated these SolrClient setters useMultiPartPost, and followRedirects. (Eric Pugh, David Smiley, Kevin Risden)+* SOLR-15787: FileSystemConfigSetService: implement the abstraction completely. It could be useful+ for putting ConfigSets on a shared file system. (Nazerke Seidan, David Smiley)+Optimizations---------------------diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.javaindex 44ea1a8b78a..b67d4e7cc50 100644--- a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java+++ b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java@@ -199,7 +199,7 @@ public class ZkConfigSetService extends ConfigSetService {throws IOException {String filePath = CONFIGS_ZKNODE + "/" + configName + "/" + fileName;try {- // if createNew is true then zkClient#makePath failOnExists is set to false+ // if overwriteOnExists is true then zkClient#makePath failOnExists is set to falsezkClient.makePath(filePath, data, CreateMode.PERSISTENT, null, !overwriteOnExists, true);} catch (KeeperException.NodeExistsException nodeExistsException) {throw new SolrException(diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java b/solr/core/src/java/org/apache/solr/core/ConfigSetService.javaindex 15896926dba..5d9fbf30054 100644--- a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java+++ b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java@@ -395,8 +395,8 @@ public abstract class ConfigSetService {public abstract void uploadConfig(String configName, Path dir) throws IOException;/**- * Upload a file to config If file does not exist, it will be uploaded If createNew param is set- * to true then file be overwritten+ * Upload a file to config If file does not exist, it will be uploaded If overwriteOnExists is set+ * to true then file will be overwritten** @param configName the name to give the config* @param fileName the name of the filediff --git a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.javaindex d8ee6325779..01dcdf8f524 100644--- a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java+++ b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java@@ -16,21 +16,31 @@*/package org.apache.solr.core;+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;+import java.io.FileNotFoundException;import java.io.IOException;import java.lang.invoke.MethodHandles;+import java.nio.file.FileVisitResult;import java.nio.file.Files;+import java.nio.file.NoSuchFileException;import java.nio.file.Path;+import java.nio.file.SimpleFileVisitor;+import java.nio.file.attribute.BasicFileAttributes;+import java.util.ArrayList;+import java.util.Collections;import java.util.List;import java.util.Map;+import java.util.Objects;import java.util.stream.Collectors;import java.util.stream.Stream;import org.apache.solr.common.SolrException;+import org.apache.solr.common.util.Utils;import org.slf4j.Logger;import org.slf4j.LoggerFactory;/**- * Solr Standalone File System ConfigSetService impl.+ * FileSystem ConfigSetService impl.** <p>Loads a ConfigSet defined by the core's configSet property, looking for a directory named for* the configSet property value underneath a base directory. If no configSet property is set, loads@@ -38,6 +48,9 @@ import org.slf4j.LoggerFactory;*/public class FileSystemConfigSetService extends ConfigSetService {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());+ /** .metadata.json hidden file where metadata is stored */+ public static final String METADATA_FILE = ".metadata.json";+private final Path configSetBase;public FileSystemConfigSetService(CoreContainer cc) {@@ -45,6 +58,12 @@ public class FileSystemConfigSetService extends ConfigSetService {this.configSetBase = cc.getConfig().getConfigSetBaseDirectory();}+ /** Testing purpose */+ protected FileSystemConfigSetService(Path configSetBase) {+ super(null, false);+ this.configSetBase = configSetBase;+ }+@Overridepublic SolrResourceLoader createCoreResourceLoader(CoreDescriptor cd) {Path instanceDir = locateInstanceDir(cd);@@ -60,68 +79,184 @@ public class FileSystemConfigSetService extends ConfigSetService {@Overridepublic boolean checkConfigExists(String configName) throws IOException {- Path solrConfigXmlFile = configSetBase.resolve(configName).resolve("solrconfig.xml");+ Path solrConfigXmlFile = getConfigDir(configName).resolve("solrconfig.xml");return Files.exists(solrConfigXmlFile);}@Overridepublic void deleteConfig(String configName) throws IOException {- throw new UnsupportedOperationException();+ deleteDir(getConfigDir(configName));}@Overridepublic void deleteFilesFromConfig(String configName, List<String> filesToDelete)throws IOException {- throw new UnsupportedOperationException();+ Path configDir = getConfigDir(configName);+ Objects.requireNonNull(filesToDelete);+ for (String fileName : filesToDelete) {+ Path file = configDir.resolve(fileName);+ if (Files.exists(file)) {+ if (Files.isDirectory(file)) {+ deleteDir(file);+ } else {+ Files.delete(file);+ }+ }+ }}@Overridepublic void copyConfig(String fromConfig, String toConfig) throws IOException {- throw new UnsupportedOperationException();+ Path source = getConfigDir(fromConfig);+ Path dest = getConfigDir(toConfig);+ copyRecursively(source, dest);+ }++ private void deleteDir(Path dir) throws IOException {+ try {+ Files.walkFileTree(+ dir,+ new SimpleFileVisitor<Path>() {+ @Override+ public FileVisitResult visitFile(Path path, BasicFileAttributes attrs)+ throws IOException {+ Files.delete(path);+ return FileVisitResult.CONTINUE;+ }++ @Override+ public FileVisitResult postVisitDirectory(Path dir, IOException ioException)+ throws IOException {+ Files.delete(dir);+ return FileVisitResult.CONTINUE;+ }+ });+ } catch (NoSuchFileException e) {+ // do nothing+ }}@Override- public void uploadConfig(String configName, Path dir) throws IOException {- throw new UnsupportedOperationException();+ public void uploadConfig(String configName, Path source) throws IOException {+ Path dest = getConfigDir(configName);+ copyRecursively(source, dest);}@Overridepublic void uploadFileToConfig(String configName, String fileName, byte[] data, boolean overwriteOnExists)throws IOException {- throw new UnsupportedOperationException();+ Path filePath = getConfigDir(configName).resolve(fileName);+ if (!Files.exists(filePath) || overwriteOnExists) {+ Files.write(filePath, data);+ }}@Overridepublic void setConfigMetadata(String configName, Map<String, Object> data) throws IOException {- throw new UnsupportedOperationException();+ // store metadata in .metadata.json file+ Path metadataPath = getConfigDir(configName).resolve(METADATA_FILE);+ Files.write(metadataPath, Utils.toJSON(data));}@Overridepublic Map<String, Object> getConfigMetadata(String configName) throws IOException {- throw new UnsupportedOperationException();+ // get metadata from .metadata.json file+ Path metadataPath = getConfigDir(configName).resolve(METADATA_FILE);+ byte[] data = null;+ try {+ data = Files.readAllBytes(metadataPath);+ } catch (NoSuchFileException e) {+ return Collections.emptyMap();+ }+ @SuppressWarnings("unchecked")+ Map<String, Object> metadata = (Map<String, Object>) Utils.fromJSON(data);+ return metadata;}@Override- public void downloadConfig(String configName, Path dir) throws IOException {- throw new UnsupportedOperationException();+ public void downloadConfig(String configName, Path dest) throws IOException {+ Path source = getConfigDir(configName);+ copyRecursively(source, dest);+ }++ private void copyRecursively(Path source, Path target) throws IOException {+ try {+ Files.walkFileTree(+ source,+ new SimpleFileVisitor<Path>() {+ @Override+ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)+ throws IOException {+ Files.createDirectories(target.resolve(source.relativize(dir).toString()));+ return FileVisitResult.CONTINUE;+ }++ @Override+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)+ throws IOException {+ Files.copy(+ file, target.resolve(source.relativize(file).toString()), REPLACE_EXISTING);+ return FileVisitResult.CONTINUE;+ }+ });+ } catch (NoSuchFileException e) {+ // do nothing+ }}@Overridepublic List<String> listConfigs() throws IOException {try (Stream<Path> configs = Files.list(configSetBase)) {- return configs.map(Path::getFileName).map(Path::toString).collect(Collectors.toList());+ return configs+ .map(Path::getFileName)+ .map(Path::toString)+ .sorted()+ .collect(Collectors.toList());}}@Override- public byte[] downloadFileFromConfig(String configName, String filePath) throws IOException {- throw new UnsupportedOperationException();+ public byte[] downloadFileFromConfig(String configName, String fileName) throws IOException {+ Path filePath = getConfigDir(configName).resolve(fileName);+ byte[] data = null;+ try {+ data = Files.readAllBytes(filePath);+ } catch (NoSuchFileException e) {+ // do nothing+ }+ return data;}@Overridepublic List<String> getAllConfigFiles(String configName) throws IOException {- throw new UnsupportedOperationException();+ Path configDir = getConfigDir(configName);+ List<String> filePaths = new ArrayList<>();+ Files.walkFileTree(+ configDir,+ new SimpleFileVisitor<Path>() {+ @Override+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)+ throws IOException {+ // don't include hidden (.) files+ if (!Files.isHidden(file)) {+ filePaths.add(configDir.relativize(file).toString());+ return FileVisitResult.CONTINUE;+ }+ return FileVisitResult.CONTINUE;+ }++ @Override+ public FileVisitResult postVisitDirectory(Path dir, IOException ioException) {+ String relativePath = configDir.relativize(dir).toString();+ if (!relativePath.isEmpty()) {+ filePaths.add(relativePath + "/");+ }+ return FileVisitResult.CONTINUE;+ }+ });+ Collections.sort(filePaths);+ return filePaths;}protected Path locateInstanceDir(CoreDescriptor cd) {@@ -136,8 +271,8 @@ public class FileSystemConfigSetService extends ConfigSetService {}@Override- protected Long getCurrentSchemaModificationVersion(- String configSet, SolrConfig solrConfig, String schemaFileName) throws IOException {+ public Long getCurrentSchemaModificationVersion(+ String configSet, SolrConfig solrConfig, String schemaFileName) {Path schemaFile = solrConfig.getResourceLoader().getConfigPath().resolve(schemaFileName);try {return Files.getLastModifiedTime(schemaFile).toMillis();@@ -148,4 +283,13 @@ public class FileSystemConfigSetService extends ConfigSetService {return null; // debatable; we'll see an error soon if there's a real problem}}++ protected Path getConfigDir(String configName) throws IOException {+ // startsWith works simply; we must normalize()+ Path path = configSetBase.resolve(configName).normalize();+ if (!path.startsWith(configSetBase)) {+ throw new IOException("configName=" + configName + " is not found under configSetBase dir");+ }+ return path;+ }}diff --git a/solr/core/src/test/org/apache/solr/core/TestConfigSetService.java b/solr/core/src/test/org/apache/solr/core/TestConfigSetService.javanew file mode 100644index 00000000000..07cc09b91a2--- /dev/null+++ b/solr/core/src/test/org/apache/solr/core/TestConfigSetService.java@@ -0,0 +1,130 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.core;++import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;+import java.io.IOException;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;+import java.util.Arrays;+import java.util.Collections;+import java.util.List;+import java.util.Map;+import java.util.function.Supplier;+import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.cloud.ZkConfigSetService;+import org.apache.solr.cloud.ZkTestServer;+import org.apache.solr.common.cloud.SolrZkClient;+import org.junit.AfterClass;+import org.junit.BeforeClass;+import org.junit.Test;++public class TestConfigSetService extends SolrTestCaseJ4 {++ private final ConfigSetService configSetService;+ private static ZkTestServer zkServer;+ private static SolrZkClient zkClient;++ @BeforeClass+ public static void startZkServer() throws Exception {+ zkServer = new ZkTestServer(createTempDir("zkData"));+ zkServer.run();+ zkClient = new SolrZkClient(zkServer.getZkAddress("/solr"), 10000);+ }++ @AfterClass+ public static void shutdownZkServer() throws IOException, InterruptedException {+ zkClient.close();+ if (null != zkServer) {+ zkServer.shutdown();+ }+ zkServer = null;+ }++ public TestConfigSetService(Supplier<ConfigSetService> configSetService) {+ this.configSetService = configSetService.get();+ }++ @ParametersFactory+ @SuppressWarnings("rawtypes")+ public static Iterable<Supplier[]> parameters() {+ return Arrays.asList(+ new Supplier[][] {+ {() -> new ZkConfigSetService(zkClient)},+ {() -> new FileSystemConfigSetService(createTempDir("configsets"))}+ });+ }++ @Test+ public void testConfigSetServiceOperations() throws IOException {+ final String configName = "testconfig";+ byte[] testdata = "test data".getBytes(StandardCharsets.UTF_8);++ Path configDir = createTempDir("testconfig");+ Files.createFile(configDir.resolve("solrconfig.xml"));+ Files.write(configDir.resolve("file1"), testdata);+ Files.createFile(configDir.resolve("file2"));+ Files.createDirectory(configDir.resolve("subdir"));+ Files.createFile(configDir.resolve("subdir").resolve("file3"));++ configSetService.uploadConfig(configName, configDir);++ assertTrue(configSetService.checkConfigExists(configName));+ assertFalse(configSetService.checkConfigExists("dummyConfig"));++ byte[] data = "file3 data".getBytes(StandardCharsets.UTF_8);+ configSetService.uploadFileToConfig(configName, "subdir/file3", data, true);+ assertArrayEquals(configSetService.downloadFileFromConfig(configName, "subdir/file3"), data);++ data = "file4 data".getBytes(StandardCharsets.UTF_8);+ configSetService.uploadFileToConfig(configName, "subdir/file4", data, true);+ assertArrayEquals(configSetService.downloadFileFromConfig(configName, "subdir/file4"), data);++ Map<String, Object> metadata = configSetService.getConfigMetadata(configName);+ assertTrue(metadata.isEmpty());++ configSetService.setConfigMetadata(configName, Collections.singletonMap("trusted", true));+ metadata = configSetService.getConfigMetadata(configName);+ assertTrue(metadata.containsKey("trusted"));++ configSetService.setConfigMetadata(configName, Collections.singletonMap("foo", true));+ assertFalse(configSetService.getConfigMetadata(configName).containsKey("trusted"));+ assertTrue(configSetService.getConfigMetadata(configName).containsKey("foo"));++ List<String> configFiles = configSetService.getAllConfigFiles(configName);+ assertEquals(+ configFiles.toString(),+ "[file1, file2, solrconfig.xml, subdir/, subdir/file3, subdir/file4]");++ List<String> configs = configSetService.listConfigs();+ assertEquals(configs.toString(), "[testconfig]");++ configSetService.copyConfig(configName, "testconfig.AUTOCREATED");+ List<String> copiedConfigFiles = configSetService.getAllConfigFiles("testconfig.AUTOCREATED");+ assertEquals(configFiles.toString(), (copiedConfigFiles.toString()));++ assertEquals(2, configSetService.listConfigs().size());++ configSetService.deleteConfig("testconfig.AUTOCREATED");+ assertFalse(configSetService.checkConfigExists("testconfig.AUTOCREATED"));++ configSetService.deleteConfig(configName);+ assertFalse(configSetService.checkConfigExists(configName));+ }+}diff --git a/solr/core/src/test/org/apache/solr/core/TestFileSystemConfigSetService.java b/solr/core/src/test/org/apache/solr/core/TestFileSystemConfigSetService.javanew file mode 100644index 00000000000..7b0e26d47cc--- /dev/null+++ b/solr/core/src/test/org/apache/solr/core/TestFileSystemConfigSetService.java@@ -0,0 +1,115 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.core;++import static org.apache.solr.core.FileSystemConfigSetService.METADATA_FILE;++import java.io.IOException;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;+import java.util.List;+import java.util.Map;+import java.util.stream.Collectors;+import java.util.stream.Stream;+import org.apache.commons.io.file.PathUtils;+import org.apache.solr.SolrTestCaseJ4;+import org.junit.AfterClass;+import org.junit.BeforeClass;+import org.junit.Test;++public class TestFileSystemConfigSetService extends SolrTestCaseJ4 {+ private static Path configSetBase;+ private static FileSystemConfigSetService fileSystemConfigSetService;++ @BeforeClass+ public static void beforeClass() throws Exception {+ configSetBase = createTempDir();+ fileSystemConfigSetService = new FileSystemConfigSetService(configSetBase);+ }++ @AfterClass+ public static void afterClass() throws Exception {+ PathUtils.deleteDirectory(configSetBase);+ fileSystemConfigSetService = null;+ }++ @Test+ public void testUploadAndDeleteConfig() throws IOException {+ String configName = "testconfig";++ fileSystemConfigSetService.uploadConfig(configName, configset("cloud-minimal"));++ assertEquals(fileSystemConfigSetService.listConfigs().size(), 1);+ assertTrue(fileSystemConfigSetService.checkConfigExists(configName));++ byte[] testdata = "test data".getBytes(StandardCharsets.UTF_8);+ fileSystemConfigSetService.uploadFileToConfig(configName, "testfile", testdata, true);++ // metadata is stored in .metadata.json+ fileSystemConfigSetService.setConfigMetadata(configName, Map.of("key1", "val1"));+ Map<String, Object> metadata = fileSystemConfigSetService.getConfigMetadata(configName);+ assertEquals(metadata.toString(), "{key1=val1}");++ List<String> allConfigFiles = fileSystemConfigSetService.getAllConfigFiles(configName);+ assertEquals(allConfigFiles.toString(), "[schema.xml, solrconfig.xml, testfile]");++ fileSystemConfigSetService.deleteFilesFromConfig(+ configName, List.of(METADATA_FILE, "testfile"));+ metadata = fileSystemConfigSetService.getConfigMetadata(configName);+ assertTrue(metadata.isEmpty());++ allConfigFiles = fileSystemConfigSetService.getAllConfigFiles(configName);+ assertEquals(allConfigFiles.toString(), "[schema.xml, solrconfig.xml]");++ fileSystemConfigSetService.copyConfig(configName, "copytestconfig");+ assertEquals(fileSystemConfigSetService.listConfigs().size(), 2);++ allConfigFiles = fileSystemConfigSetService.getAllConfigFiles("copytestconfig");+ assertEquals(allConfigFiles.toString(), "[schema.xml, solrconfig.xml]");++ Path downloadConfig = createTempDir("downloadConfig");+ fileSystemConfigSetService.downloadConfig(configName, downloadConfig);++ List<String> configs = getFileList(downloadConfig);+ assertEquals(configs.toString(), "[schema.xml, solrconfig.xml]");++ Exception ex =+ assertThrows(+ IOException.class,+ () -> {+ fileSystemConfigSetService.uploadConfig("../dummy", createTempDir("tmp"));+ });+ assertTrue(ex.getMessage().startsWith("configName=../dummy is not found under configSetBase"));++ fileSystemConfigSetService.deleteConfig(configName);+ fileSystemConfigSetService.deleteConfig("copytestconfig");++ assertFalse(fileSystemConfigSetService.checkConfigExists(configName));+ assertFalse(fileSystemConfigSetService.checkConfigExists("copytestconfig"));+ }++ private static List<String> getFileList(Path confDir) throws IOException {+ try (Stream<Path> configs = Files.list(confDir)) {+ return configs+ .map(Path::getFileName)+ .map(Path::toString)+ .sorted()+ .collect(Collectors.toList());+ }+ }+}