Apache Solr open-source search software
SOLR-14907: Adding V2 API for ConfigSet Upload. (#1996)
Houston Putman5091e75c9d85
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 8861d97a1ae..8dc504afd78 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -149,7 +149,8 @@ Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this rNew Features----------------------(No changes)++* SOLR-14907: Add v2 API for configSet upload, including single-file insertion. (Houston Putman)Improvements---------------------diff --git a/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java b/solr/core/src/java/org/apache/solr/handler/ClusterAPI.javaindex 81ab2b72c79..f3f28161b4e 100644--- a/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java+++ b/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java@@ -47,6 +47,7 @@ import org.apache.solr.response.SolrQueryResponse;import static org.apache.solr.client.solrj.SolrRequest.METHOD.DELETE;import static org.apache.solr.client.solrj.SolrRequest.METHOD.GET;import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST;+import static org.apache.solr.client.solrj.SolrRequest.METHOD.PUT;import static org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.REQUESTID;import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDROLE;import static org.apache.solr.common.params.CollectionParams.CollectionAction.CLUSTERPROP;@@ -133,6 +134,37 @@ public class ClusterAPI {}+ @EndPoint(method = PUT,+ path = "/cluster/configs/{name}",+ permission = CONFIG_EDIT_PERM+ )+ public void uploadConfigSet(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {+ req = wrapParams(req,+ "action", ConfigSetParams.ConfigSetAction.UPLOAD.toString(),+ CommonParams.NAME, req.getPathTemplateValues().get("name"),+ ConfigSetParams.OVERWRITE, true,+ ConfigSetParams.CLEANUP, false);+ configSetsHandler.handleRequestBody(req, rsp);+ }++ @EndPoint(method = PUT,+ path = "/cluster/configs/{name}/*",+ permission = CONFIG_EDIT_PERM+ )+ public void insertIntoConfigSet(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {+ String path = req.getPathTemplateValues().get("*");+ if (path == null || path.isBlank()) {+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "In order to insert a file in a configSet, a filePath must be provided in the url after the name of the configSet.");+ }+ req = wrapParams(req,+ "action", ConfigSetParams.ConfigSetAction.UPLOAD.toString(),+ CommonParams.NAME, req.getPathTemplateValues().get("name"),+ ConfigSetParams.FILE_PATH, path,+ ConfigSetParams.OVERWRITE, true,+ ConfigSetParams.CLEANUP, false);+ configSetsHandler.handleRequestBody(req, rsp);+ }+@SuppressWarnings({"rawtypes"})public static SolrQueryRequest wrapParams(SolrQueryRequest req, Object... def) {Map m = Utils.makeMap(def);diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.javaindex 736aeb7b56e..cb6d2a3de85 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java@@ -68,6 +68,7 @@ import static org.apache.solr.common.params.CommonParams.NAME;import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.CREATE;import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.DELETE;import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.LIST;+import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.UPLOAD;/*** A {@link org.apache.solr.request.SolrRequestHandler} for ConfigSets API requests.@@ -193,10 +194,10 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionNfixedSingleFilePath = fixedSingleFilePath.substring(1);}if (fixedSingleFilePath.isEmpty()) {- throw new SolrException(ErrorCode.BAD_REQUEST, "The filePath provided for upload, '" + singleFilePath + "', is not valid.");+ throw new SolrException(ErrorCode.BAD_REQUEST, "The file path provided for upload, '" + singleFilePath + "', is not valid.");} else if (cleanup) {// Cleanup is not allowed while using singleFilePath upload- throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet uploads do not allow cleanup=true when filePath is used.");+ throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet uploads do not allow cleanup=true when file path is used.");} else {try {// Create a node for the configuration in zookeeper@@ -206,7 +207,7 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionNzkClient.makePath(filePathInZk, IOUtils.toByteArray(inputStream), CreateMode.PERSISTENT, null, !allowOverwrite, true);} catch(KeeperException.NodeExistsException nodeExistsException) {throw new SolrException(ErrorCode.BAD_REQUEST,- "The path " + singleFilePath + " for configSet " + configSetName + " already exists. In order to overwrite, provide overwrite=true.");+ "The path " + singleFilePath + " for configSet " + configSetName + " already exists. In order to overwrite, provide overwrite=true or use an HTTP PUT with the V2 API.");}}return;@@ -230,7 +231,9 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionNZipInputStream zis = new ZipInputStream(inputStream, StandardCharsets.UTF_8);ZipEntry zipEntry = null;+ boolean hasEntry = false;while ((zipEntry = zis.getNextEntry()) != null) {+ hasEntry = true;String filePathInZk = configPathInZk + "/" + zipEntry.getName();if (filePathInZk.endsWith("/")) {filesToDelete.remove(filePathInZk.substring(0, filePathInZk.length() -1));@@ -245,6 +248,10 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN}}zis.close();+ if (!hasEntry) {+ throw new SolrException(ErrorCode.BAD_REQUEST,+ "Either empty zipped data, or non-zipped data was uploaded. In order to upload a configSet, you must zip a non-empty directory to upload.");+ }deleteUnusedFiles(zkClient, filesToDelete);// If the request is doing a full trusted overwrite of an untrusted configSet (overwrite=true, cleanup=true), then trust the configSet.@@ -400,6 +407,13 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN}public enum ConfigSetOperation {+ UPLOAD_OP(UPLOAD) {+ @Override+ public Map<String, Object> call(SolrQueryRequest req, SolrQueryResponse rsp, ConfigSetsHandler h) throws Exception {+ h.handleConfigUploadRequest(req, rsp);+ return null;+ }+ },CREATE_OP(CREATE) {@Overridepublic Map<String, Object> call(SolrQueryRequest req, SolrQueryResponse rsp, ConfigSetsHandler h) throws Exception {diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.javaindex c7b0a82466c..65b09d56e7d 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java@@ -52,7 +52,9 @@ import com.google.common.collect.ImmutableMap;import org.apache.commons.io.FileUtils;import org.apache.http.HttpEntity;import org.apache.http.auth.BasicUserPrincipal;+import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;import org.apache.http.client.methods.HttpPost;+import org.apache.http.client.methods.HttpPut;import org.apache.http.entity.ByteArrayEntity;import org.apache.http.message.BasicHeader;import org.apache.http.util.EntityUtils;@@ -343,7 +345,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {@SuppressWarnings({"rawtypes"})Map map = postDataAndGetResponse(cluster.getSolrClient(),cluster.getJettySolrRunners().get(0).getBaseUrl().toString()- + "/admin/configs?action=UPLOAD", emptyData, null);+ + "/admin/configs?action=UPLOAD", emptyData, null, false);assertNotNull(map);unIgnoreException("The configuration name should be provided");long statusCode = (long) getObjectByPath(map, false,@@ -364,7 +366,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {ignoreException("already exists");map = postDataAndGetResponse(cluster.getSolrClient(),cluster.getJettySolrRunners().get(0).getBaseUrl().toString()- + "/admin/configs?action=UPLOAD&name=myconf", emptyData, null);+ + "/admin/configs?action=UPLOAD&name=myconf", emptyData, null, false);assertNotNull(map);unIgnoreException("already exists`");statusCode = (long) getObjectByPath(map, false,@@ -380,7 +382,16 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {}@Test- public void testUploadDisabled() throws Exception {+ public void testUploadDisabledV1() throws Exception {+ testUploadDisabled(false);+ }++ @Test+ public void testUploadDisabledV2() throws Exception {+ testUploadDisabled(true);+ }++ public void testUploadDisabled(boolean v2) throws Exception {try (SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(),AbstractZkTestCase.TIMEOUT, 45000, null)) {@@ -388,7 +399,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {for (boolean enabled: new boolean[] {true, false}) {System.setProperty("configset.upload.enabled", String.valueOf(enabled));try {- long statusCode = uploadConfigSet("regular", "test-enabled-is-" + enabled, null, zkClient);+ long statusCode = uploadConfigSet("regular", "test-enabled-is-" + enabled, null, zkClient, v2);assertEquals("ConfigSet upload enabling/disabling not working as expected for enabled=" + enabled + ".",enabled? 0l: 400l, statusCode);} finally {@@ -400,20 +411,29 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {}@Test- public void testOverwrite() throws Exception {+ public void testOverwriteV1() throws Exception {+ testOverwrite(false);+ }++ @Test+ public void testOverwriteV2() throws Exception {+ testOverwrite(true);+ }++ public void testOverwrite(boolean v2) throws Exception {String configsetName = "regular";- String configsetSuffix = "testOverwrite-1";+ String configsetSuffix = "testOverwrite-1-" + v2;uploadConfigSetWithAssertions(configsetName, configsetSuffix, null);try (SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(),AbstractZkTestCase.TIMEOUT, 45000, null)) {int solrconfigZkVersion = getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml");ignoreException("The configuration regulartestOverwrite-1 already exists in zookeeper");assertEquals("Can't overwrite an existing configset unless the overwrite parameter is set",- 400, uploadConfigSet(configsetName, configsetSuffix, null, zkClient, false, false));+ 400, uploadConfigSet(configsetName, configsetSuffix, null, false, false, v2));unIgnoreException("The configuration regulartestOverwrite-1 already exists in zookeeper");assertEquals("Expecting version to remain equal",solrconfigZkVersion, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"));- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, zkClient, true, false));+ assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2));assertTrue("Expecting version bump",solrconfigZkVersion < getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"));}@@ -421,25 +441,35 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {}@Test- public void testOverwriteWithCleanup() throws Exception {+ public void testOverwriteWithCleanupV1() throws Exception {+ testOverwriteWithCleanup(false);+ }++ @Test+ public void testOverwriteWithCleanupV2() throws Exception {+ testOverwriteWithCleanup(true);+ }++ public void testOverwriteWithCleanup(boolean v2) throws Exception {String configsetName = "regular";- String configsetSuffix = "testOverwriteWithCleanup-1";+ String configsetSuffix = "testOverwriteWithCleanup-1-" + v2;uploadConfigSetWithAssertions(configsetName, configsetSuffix, null);try (SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(),AbstractZkTestCase.TIMEOUT, 45000, null)) {+ String configPath = "/configs/" + configsetName + configsetSuffix;List<String> extraFiles = Arrays.asList(- "/configs/regulartestOverwriteWithCleanup-1/foo1",- "/configs/regulartestOverwriteWithCleanup-1/foo2",- "/configs/regulartestOverwriteWithCleanup-1/foo2/1",- "/configs/regulartestOverwriteWithCleanup-1/foo2/2");+ configPath + "/foo1",+ configPath + "/foo2",+ configPath + "/foo2/1",+ configPath + "/foo2/2");for (String f : extraFiles) {zkClient.makePath(f, true);}- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, zkClient, true, false));+ assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2));for (String f : extraFiles) {assertTrue("Expecting file " + f + " to exist in ConfigSet but it's gone", zkClient.exists(f, true));}- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, zkClient, true, true));+ assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2));for (String f : extraFiles) {assertFalse("Expecting file " + f + " to be deleted from ConfigSet but it wasn't", zkClient.exists(f, true));}@@ -448,38 +478,49 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {}@Test- public void testOverwriteWithTrust() throws Exception {+ public void testOverwriteWithTrustV1() throws Exception {+ testOverwriteWithTrust(false);+ }++ @Test+ public void testOverwriteWithTrustV2() throws Exception {+ testOverwriteWithTrust(true);+ }++ public void testOverwriteWithTrust(boolean v2) throws Exception {String configsetName = "regular";- String configsetSuffix = "testOverwriteWithTrust-1";+ String configsetSuffix = "testOverwriteWithTrust-1-" + v2;uploadConfigSetWithAssertions(configsetName, configsetSuffix, null);try (SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(),AbstractZkTestCase.TIMEOUT, 45000, null)) {assertFalse(isTrusted(zkClient, configsetName, configsetSuffix));int solrconfigZkVersion = getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml");// Was untrusted, overwrite with untrusted- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, zkClient, true, false));+ assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2));assertTrue("Expecting version bump",solrconfigZkVersion < getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"));assertFalse(isTrusted(zkClient, configsetName, configsetSuffix));solrconfigZkVersion = getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml");// Was untrusted, overwrite with trusted but no cleanup- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr", zkClient, true, false));+ assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2));assertTrue("Expecting version bump",solrconfigZkVersion < getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"));assertFalse(isTrusted(zkClient, configsetName, configsetSuffix));solrconfigZkVersion = getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml");// Was untrusted, overwrite with trusted with cleanup but fail on unzipping.- // Should not set trusted=true- assertEquals(500, uploadBadConfigSet(configsetName, configsetSuffix, "solr", zkClient, true, true));+ // Should not set trusted=true in configSet+ ignoreException("Either empty zipped data, or non-zipped data was passed. In order to upload a configSet, you must zip a non-empty directory to upload.");+ assertEquals(400, uploadBadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2));assertEquals("Expecting version bump",solrconfigZkVersion, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"));assertFalse(isTrusted(zkClient, configsetName, configsetSuffix));solrconfigZkVersion = getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml");+ ignoreException("Either empty zipped data, or non-zipped data was passed. In order to upload a configSet, you must zip a non-empty directory to upload.");// Was untrusted, overwrite with trusted with cleanup- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr", zkClient, true, true));+ assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2));assertTrue("Expecting version bump",solrconfigZkVersion < getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"));assertTrue(isTrusted(zkClient, configsetName, configsetSuffix));@@ -488,7 +529,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {// Was trusted, try to overwrite with untrusted with no cleanupignoreException("Trying to make an unstrusted ConfigSet update on a trusted configSet");assertEquals("Can't upload a trusted configset with an untrusted request",- 400, uploadConfigSet(configsetName, configsetSuffix, null, zkClient, true, false));+ 400, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2));assertEquals("Expecting version to remain equal",solrconfigZkVersion, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"));assertTrue(isTrusted(zkClient, configsetName, configsetSuffix));@@ -496,21 +537,21 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {// Was trusted, try to overwrite with untrusted with cleanupignoreException("Trying to make an unstrusted ConfigSet update on a trusted configSet");assertEquals("Can't upload a trusted configset with an untrusted request",- 400, uploadConfigSet(configsetName, configsetSuffix, null, zkClient, true, true));+ 400, uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2));assertEquals("Expecting version to remain equal",solrconfigZkVersion, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"));assertTrue(isTrusted(zkClient, configsetName, configsetSuffix));unIgnoreException("Trying to make an unstrusted ConfigSet update on a trusted configSet");// Was trusted, overwrite with trusted no cleanup- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr", zkClient, true, false));+ assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2));assertTrue("Expecting version bump",solrconfigZkVersion < getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"));assertTrue(isTrusted(zkClient, configsetName, configsetSuffix));solrconfigZkVersion = getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml");// Was trusted, overwrite with trusted with cleanup- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr", zkClient, true, true));+ assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2));assertTrue("Expecting version bump",solrconfigZkVersion < getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"));assertTrue(isTrusted(zkClient, configsetName, configsetSuffix));@@ -519,47 +560,74 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {}@Test- public void testSingleFileOverwrite() throws Exception {+ public void testSingleFileOverwriteV1() throws Exception {+ testSingleFileOverwrite(false);+ }++ @Test+ public void testSingleFileOverwriteV2() throws Exception {+ testSingleFileOverwrite(true);+ }++ public void testSingleFileOverwrite(boolean v2) throws Exception {String configsetName = "regular";- String configsetSuffix = "testSinglePathOverwrite-1";+ String configsetSuffix = "testSinglePathOverwrite-1-" + v2;uploadConfigSetWithAssertions(configsetName, configsetSuffix, null);try (SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(),AbstractZkTestCase.TIMEOUT, 45000, null)) {int solrconfigZkVersion = getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml");ignoreException("The configuration regulartestOverwrite-1 already exists in zookeeper");assertEquals("Can't overwrite an existing configset unless the overwrite parameter is set",- 400, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, zkClient, "solr/configsets/upload/regular/solrconfig.xml", "solrconfig.xml", false, false));+ 400, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, "solr/configsets/upload/regular/solrconfig.xml", "solrconfig.xml", false, false, v2));unIgnoreException("The configuration regulartestOverwrite-1 already exists in zookeeper");assertEquals("Expecting version to remain equal",solrconfigZkVersion, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"));- assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, zkClient, "solr/configsets/upload/regular/solrconfig.xml", "solrconfig.xml", true, false));+ assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, "solr/configsets/upload/regular/solrconfig.xml", "solrconfig.xml", true, false, v2));assertTrue("Expecting version bump",solrconfigZkVersion < getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"));}}@Test- public void testNewSingleFile() throws Exception {+ public void testNewSingleFileV1() throws Exception {+ testNewSingleFile(false);+ }++ @Test+ public void testNewSingleFileV2() throws Exception {+ testNewSingleFile(true);+ }++ public void testNewSingleFile(boolean v2) throws Exception {String configsetName = "regular";- String configsetSuffix = "testSinglePathNew-1";+ String configsetSuffix = "testSinglePathNew-1-" + v2;uploadConfigSetWithAssertions(configsetName, configsetSuffix, null);try (SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(),AbstractZkTestCase.TIMEOUT, 45000, null)) {- assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, zkClient, "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", false, false));+ assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", false, false, v2));assertEquals("Expecting first version of new file", 0, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/upload/path/solrconfig.xml"));assertConfigsetFiles(configsetName, configsetSuffix, zkClient);}}@Test- public void testSingleWithCleanup() throws Exception {+ public void testSingleWithCleanupV1() throws Exception {+ testSingleWithCleanup(false);+ }++ @Test+ public void testSingleWithCleanupV2() throws Exception {+ testSingleWithCleanup(true);+ }++ public void testSingleWithCleanup(boolean v2) throws Exception {String configsetName = "regular";- String configsetSuffix = "testSinglePathCleanup-1";+ String configsetSuffix = "testSinglePathCleanup-1-" + v2;uploadConfigSetWithAssertions(configsetName, configsetSuffix, null);try (SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(),AbstractZkTestCase.TIMEOUT, 45000, null)) {ignoreException("ConfigSet uploads do not allow cleanup=true when filePath is used.");- assertEquals(400, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, zkClient, "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", false, true));+ assertEquals(400, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", true, true, v2));assertFalse("New file should not exist, since the trust check did not succeed.", zkClient.exists("/configs/"+configsetName+configsetSuffix+"/test/upload/path/solrconfig.xml", true));assertConfigsetFiles(configsetName, configsetSuffix, zkClient);unIgnoreException("ConfigSet uploads do not allow cleanup=true when filePath is used.");@@ -567,20 +635,29 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {}@Test- public void testSingleFileTrusted() throws Exception {+ public void testSingleFileTrustedV1() throws Exception {+ testSingleFileTrusted(false);+ }++ @Test+ public void testSingleFileTrustedV2() throws Exception {+ testSingleFileTrusted(true);+ }++ public void testSingleFileTrusted(boolean v2) throws Exception {String configsetName = "regular";- String configsetSuffix = "testSinglePathTrusted-1";+ String configsetSuffix = "testSinglePathTrusted-1-" + v2;uploadConfigSetWithAssertions(configsetName, configsetSuffix, "solr");try (SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(),AbstractZkTestCase.TIMEOUT, 45000, null)) {- assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, "solr", zkClient, "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", true, false));+ assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, "solr", "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", true, false, v2));assertEquals("Expecting first version of new file", 0, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/upload/path/solrconfig.xml"));assertTrue(isTrusted(zkClient, configsetName, configsetSuffix));assertConfigsetFiles(configsetName, configsetSuffix, zkClient);ignoreException("Trying to make an unstrusted ConfigSet update on a trusted configSet");assertEquals("Can't upload a trusted configset with an untrusted request",- 400, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, zkClient, "solr/configsets/upload/regular/solrconfig.xml", "/test/different/path/solrconfig.xml", true, false));+ 400, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, "solr/configsets/upload/regular/solrconfig.xml", "/test/different/path/solrconfig.xml", true, false, v2));assertFalse("New file should not exist, since the trust check did not succeed.", zkClient.exists("/configs/"+configsetName+configsetSuffix+"/test/different/path/solrconfig.xml", true));assertTrue(isTrusted(zkClient, configsetName, configsetSuffix));assertConfigsetFiles(configsetName, configsetSuffix, zkClient);@@ -589,7 +666,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {ignoreException("Trying to make an unstrusted ConfigSet update on a trusted configSet");int extraFileZkVersion = getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/upload/path/solrconfig.xml");assertEquals("Can't upload a trusted configset with an untrusted request",- 400, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, zkClient, "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", true, false));+ 400, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", true, false, v2));assertEquals("Expecting version to remain equal",extraFileZkVersion, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/upload/path/solrconfig.xml"));assertTrue(isTrusted(zkClient, configsetName, configsetSuffix));@@ -599,27 +676,36 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {}@Test- public void testSingleFileUntrusted() throws Exception {+ public void testSingleFileUntrustedV1() throws Exception {+ testSingleFileUntrusted(false);+ }++ @Test+ public void testSingleFileUntrustedV2() throws Exception {+ testSingleFileUntrusted(true);+ }++ public void testSingleFileUntrusted(boolean v2) throws Exception {String configsetName = "regular";- String configsetSuffix = "testSinglePathUntrusted-1";+ String configsetSuffix = "testSinglePathUntrusted-1-" + v2;uploadConfigSetWithAssertions(configsetName, configsetSuffix, null);try (SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(),AbstractZkTestCase.TIMEOUT, 45000, null)) {// New file with trusted request- assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, "solr", zkClient, "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", false, false));+ assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, "solr", "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", false, false, v2));assertEquals("Expecting first version of new file", 0, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/upload/path/solrconfig.xml"));assertFalse(isTrusted(zkClient, configsetName, configsetSuffix));assertConfigsetFiles(configsetName, configsetSuffix, zkClient);// New file with untrusted request- assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, zkClient, "solr/configsets/upload/regular/solrconfig.xml", "/test/different/path/solrconfig.xml", false, false));+ assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, "solr/configsets/upload/regular/solrconfig.xml", "/test/different/path/solrconfig.xml", false, false, v2));assertEquals("Expecting first version of new file", 0, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/different/path/solrconfig.xml"));assertFalse(isTrusted(zkClient, configsetName, configsetSuffix));assertConfigsetFiles(configsetName, configsetSuffix, zkClient);// Overwrite with trusted requestint extraFileZkVersion = getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/different/path/solrconfig.xml");- assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, "solr", zkClient, "solr/configsets/upload/regular/solrconfig.xml", "/test/different/path/solrconfig.xml", true, false));+ assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, "solr", "solr/configsets/upload/regular/solrconfig.xml", "/test/different/path/solrconfig.xml", true, false, v2));assertTrue("Expecting version bump",extraFileZkVersion < getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/different/path/solrconfig.xml"));assertFalse(isTrusted(zkClient, configsetName, configsetSuffix));@@ -627,7 +713,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {// Overwrite with untrusted requestextraFileZkVersion = getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/upload/path/solrconfig.xml");- assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, zkClient, "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", true, false));+ assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffix, null, "solr/configsets/upload/regular/solrconfig.xml", "/test/upload/path/solrconfig.xml", true, false, v2));assertTrue("Expecting version bump",extraFileZkVersion < getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/upload/path/solrconfig.xml"));assertFalse(isTrusted(zkClient, configsetName, configsetSuffix));@@ -636,7 +722,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {// Make sure that cleanup flag does not result in configSet being trusted.ignoreException("ConfigSet uploads do not allow cleanup=true when filePath is used.");extraFileZkVersion = getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/different/path/solrconfig.xml");- assertEquals(400, uploadSingleConfigSetFile(configsetName, configsetSuffix, "solr", zkClient, "solr/configsets/upload/regular/solrconfig.xml", "/test/different/path/solrconfig.xml", true, true));+ assertEquals(400, uploadSingleConfigSetFile(configsetName, configsetSuffix, "solr", "solr/configsets/upload/regular/solrconfig.xml", "/test/different/path/solrconfig.xml", true, true, v2));assertEquals("Expecting version to stay the same",extraFileZkVersion, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "test/different/path/solrconfig.xml"));assertFalse("The cleanup=true flag allowed for trust overwriting in a filePath upload.", isTrusted(zkClient, configsetName, configsetSuffix));@@ -646,14 +732,23 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {}@Test- public void testSingleFileNewConfig() throws Exception {+ public void testSingleFileNewConfigV1() throws Exception {+ testSingleFileNewConfig(false);+ }++ @Test+ public void testSingleFileNewConfigV2() throws Exception {+ testSingleFileNewConfig(true);+ }++ public void testSingleFileNewConfig(boolean v2) throws Exception {String configsetName = "regular";- String configsetSuffixTrusted = "testSinglePathNewConfig-1";- String configsetSuffixUntrusted = "testSinglePathNewConfig-2";+ String configsetSuffixTrusted = "testSinglePathNewConfig-1-" + v2;+ String configsetSuffixUntrusted = "testSinglePathNewConfig-2-" + v2;try (SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(),AbstractZkTestCase.TIMEOUT, 45000, null)) {// New file with trusted request- assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffixTrusted, "solr", zkClient, "solr/configsets/upload/regular/solrconfig.xml", "solrconfig.xml", false, false));+ assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffixTrusted, "solr", "solr/configsets/upload/regular/solrconfig.xml", "solrconfig.xml", false, false, v2));assertEquals("Expecting first version of new file", 0, getConfigZNodeVersion(zkClient, configsetName, configsetSuffixTrusted, "solrconfig.xml"));assertTrue(isTrusted(zkClient, configsetName, configsetSuffixTrusted));List<String> children = zkClient.getChildren(String.format(Locale.ROOT,"/configs/%s%s", configsetName, configsetSuffixTrusted), null, true);@@ -661,7 +756,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {assertEquals("Incorrect file uploaded.", "solrconfig.xml", children.get(0));// New file with trusted request- assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffixUntrusted, null, zkClient, "solr/configsets/upload/regular/solrconfig.xml", "solrconfig.xml", false, false));+ assertEquals(0, uploadSingleConfigSetFile(configsetName, configsetSuffixUntrusted, null, "solr/configsets/upload/regular/solrconfig.xml", "solrconfig.xml", false, false, v2));assertEquals("Expecting first version of new file", 0, getConfigZNodeVersion(zkClient, configsetName, configsetSuffixUntrusted, "solrconfig.xml"));assertFalse(isTrusted(zkClient, configsetName, configsetSuffixUntrusted));children = zkClient.getChildren(String.format(Locale.ROOT,"/configs/%s%s", configsetName, configsetSuffixUntrusted), null, true);@@ -762,7 +857,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {SolrZkClient zkClient = new SolrZkClient(cluster.getZkServer().getZkAddress(),AbstractZkTestCase.TIMEOUT, 45000, null);try {- long statusCode = uploadConfigSet(configSetName, suffix, username, zkClient);+ long statusCode = uploadConfigSet(configSetName, suffix, username, zkClient, true);assertEquals(0l, statusCode);assertConfigsetFiles(configSetName, suffix, zkClient);} finally {@@ -786,51 +881,68 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {}private long uploadConfigSet(String configSetName, String suffix, String username,- SolrZkClient zkClient) throws IOException {+ SolrZkClient zkClient, boolean v2) throws IOException {ZkConfigManager configManager = new ZkConfigManager(zkClient);assertFalse(configManager.configExists(configSetName + suffix));- return uploadConfigSet(configSetName, suffix, username, zkClient, false, false);+ return uploadConfigSet(configSetName, suffix, username, false, false, v2);}private long uploadConfigSet(String configSetName, String suffix, String username,- SolrZkClient zkClient, boolean overwrite, boolean cleanup) throws IOException {+ boolean overwrite, boolean cleanup, boolean v2) throws IOException {// Read zipped sample configByteBuffer sampleZippedConfig = TestSolrConfigHandler.getFileContent(createTempZipFile("solr/configsets/upload/"+configSetName), false);- @SuppressWarnings({"rawtypes"})- Map map = postDataAndGetResponse(cluster.getSolrClient(),- cluster.getJettySolrRunners().get(0).getBaseUrl().toString() + "/admin/configs?action=UPLOAD&name="+configSetName+suffix + (overwrite? "&overwrite=true" : "") + (cleanup? "&cleanup=true" : ""),- sampleZippedConfig, username);- assertNotNull(map);- long statusCode = (long) getObjectByPath(map, false, Arrays.asList("responseHeader", "status"));- return statusCode;+ return uploadGivenConfigSet(sampleZippedConfig, configSetName, suffix, username, overwrite, cleanup, v2);}private long uploadBadConfigSet(String configSetName, String suffix, String username,- SolrZkClient zkClient, boolean overwrite, boolean cleanup) throws IOException {+ boolean overwrite, boolean cleanup, boolean v2) throws IOException {// Read single file from sample configs. This should fail the unzippingByteBuffer sampleBadZippedFile = TestSolrConfigHandler.getFileContent(SolrTestCaseJ4.getFile("solr/configsets/upload/regular/solrconfig.xml").getAbsolutePath(), false);+ return uploadGivenConfigSet(sampleBadZippedFile, configSetName, suffix, username, overwrite, cleanup, v2);+ }++ private long uploadGivenConfigSet(ByteBuffer file, String configSetName, String suffix, String username,+ boolean overwrite, boolean cleanup, boolean v2) throws IOException {+ String uriEnding;+ boolean usePut = false;+ if (v2) {+ uriEnding = "/api/cluster/configs/" + configSetName+suffix + (!overwrite? "?overwrite=false" : "") + (cleanup? "?cleanup=true" : "");+ usePut = true;+ } else {+ uriEnding = "/solr/admin/configs?action=UPLOAD&name="+configSetName+suffix + (overwrite? "&overwrite=true" : "") + (cleanup? "&cleanup=true" : "");+ }+@SuppressWarnings({"rawtypes"})Map map = postDataAndGetResponse(cluster.getSolrClient(),- cluster.getJettySolrRunners().get(0).getBaseUrl().toString() + "/admin/configs?action=UPLOAD&name="+configSetName+suffix + (overwrite? "&overwrite=true" : "") + (cleanup? "&cleanup=true" : ""),- sampleBadZippedFile, username);+ cluster.getJettySolrRunners().get(0).getBaseUrl().toString().replace("/solr", "") + uriEnding,+ file, username, usePut);assertNotNull(map);long statusCode = (long) getObjectByPath(map, false, Arrays.asList("responseHeader", "status"));return statusCode;}private long uploadSingleConfigSetFile(String configSetName, String suffix, String username,- SolrZkClient zkClient, String filePath, String uploadPath, boolean overwrite, boolean cleanup) throws IOException {+ String filePath, String uploadPath, boolean overwrite, boolean cleanup, boolean v2) throws IOException {// Read single file from sample configsByteBuffer sampleConfigFile = TestSolrConfigHandler.getFileContent(SolrTestCaseJ4.getFile(filePath).getAbsolutePath(), false);+ String uriEnding;+ boolean usePut = false;+ if (v2) {+ uriEnding = "/api/cluster/configs/" + configSetName+suffix + "/" + uploadPath + (!overwrite? "?overwrite=false" : "") + (cleanup? "?cleanup=true" : "");+ usePut = true;+ } else {+ uriEnding = "/solr/admin/configs?action=UPLOAD&name="+configSetName+suffix+"&filePath="+uploadPath + (overwrite? "&overwrite=true" : "") + (cleanup? "&cleanup=true" : "");+ }+@SuppressWarnings({"rawtypes"})Map map = postDataAndGetResponse(cluster.getSolrClient(),- cluster.getJettySolrRunners().get(0).getBaseUrl().toString() + "/admin/configs?action=UPLOAD&name="+configSetName+suffix+"&filePath="+uploadPath + (overwrite? "&overwrite=true" : "") + (cleanup? "&cleanup=true" : ""),- sampleConfigFile, username);+ cluster.getJettySolrRunners().get(0).getBaseUrl().toString().replace("/solr", "") + uriEnding,+ sampleConfigFile, username, usePut);assertNotNull(map);long statusCode = (long) getObjectByPath(map, false, Arrays.asList("responseHeader", "status"));return statusCode;@@ -928,24 +1040,28 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {@SuppressWarnings({"rawtypes"})public static Map postDataAndGetResponse(CloudSolrClient cloudClient,- String uri, ByteBuffer bytarr, String username) throws IOException {- HttpPost httpPost = null;+ String uri, ByteBuffer bytarr, String username, boolean usePut) throws IOException {+ HttpEntityEnclosingRequestBase httpRequest = null;HttpEntity entity;String response = null;Map m = null;try {- httpPost = new HttpPost(uri);+ if (usePut) {+ httpRequest = new HttpPut(uri);+ } else {+ httpRequest = new HttpPost(uri);+ }if (username != null) {- httpPost.addHeader(new BasicHeader("user", username));+ httpRequest.addHeader(new BasicHeader("user", username));}- httpPost.setHeader("Content-Type", "application/octet-stream");- httpPost.setEntity(new ByteArrayEntity(bytarr.array(), bytarr+ httpRequest.setHeader("Content-Type", "application/octet-stream");+ httpRequest.setEntity(new ByteArrayEntity(bytarr.array(), bytarr.arrayOffset(), bytarr.limit()));log.info("Uploading configset with user {}", username);- entity = cloudClient.getLbClient().getHttpClient().execute(httpPost)+ entity = cloudClient.getLbClient().getHttpClient().execute(httpRequest).getEntity();try {response = EntityUtils.toString(entity, UTF_8);@@ -955,7 +1071,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {throw new AssertionError(e);}} finally {- httpPost.releaseConnection();+ httpRequest.releaseConnection();}return m;}diff --git a/solr/solr-ref-guide/src/configsets-api.adoc b/solr/solr-ref-guide/src/configsets-api.adocindex 820cb39942f..ead84f1f9f9 100644--- a/solr/solr-ref-guide/src/configsets-api.adoc+++ b/solr/solr-ref-guide/src/configsets-api.adoc@@ -101,21 +101,29 @@ The configset to be created when the upload is complete. This parameter is requi`overwrite`::If set to `true`, Solr will overwrite an existing configset with the same name (if false, the request will fail).If `filePath` is provided, then this option specifies whether the specified file within the configSet should be overwritten if it already exists.-Default is `false`.+Default is `false` when using the v1 API, but `true` when using the v2 API.`cleanup`::When overwriting an existing configset (`overwrite=true`), this parameter tells Solr to delete the files in ZooKeeper that existed in the old configset but not in the one being uploaded. Default is `false`.This parameter cannot be set to true when `filePath` is used.-filePath::+`filePath`::This parameter allows the uploading of a single, non-zipped file to the given path under the configSet in ZooKeeper.This functionality respects the `overwrite` parameter, so a request will fail if the given file path already exists in the configSet and overwrite is set to `false`.The `cleanup` parameter cannot be set to true when `filePath` is used.-The body of the request should be a zip file that contains the configset. The zip file must be created from within the `conf` directory (i.e., `solrconfig.xml` must be the top level entry in the zip file).+If uploading an entire configSet, the body of the request should be a zip file that contains the configset. The zip file must be created from within the `conf` directory (i.e., `solrconfig.xml` must be the top level entry in the zip file).Here is an example on how to create the zip file named "myconfig.zip" and upload it as a configset named "myConfigSet":+[.dynamic-tabs]+--+[example.tab-pane#v1uploadconfigset]+====+[.tab-label]*V1 API*++With the v1 API, the `upload` command must be capitalized as `UPLOAD`:+[source,bash]----$ (cd solr/server/solr/configsets/sample_techproducts_configs/conf && zip -r - *) > myconfigset.zip@@ -129,8 +137,65 @@ The same can be achieved using a Unix pipe with a single request as follows:----$ (cd server/solr/configsets/sample_techproducts_configs/conf && zip -r - *) | curl -X POST --header "Content-Type:application/octet-stream" --data-binary @- "http://localhost:8983/solr/admin/configs?action=UPLOAD&name=myConfigSet"----+====++[example.tab-pane#v2uploadconfigset]+====+[.tab-label]*V2 API*++With the v2 API, the name of the configset to upload is provided as a path parameter:++[source,bash]+----+$ (cd solr/server/solr/configsets/sample_techproducts_configs/conf && zip -r - *) > myconfigset.zip++$ curl -X PUT --header "Content-Type:application/octet-stream" --data-binary @myconfigset.zip+ "http://localhost:8983/api/cluster/configs/myConfigSet"+----++With this REST API, the default behavior is to overwrite the configSet if it already exists.+This behavior can be disabled by providing the URL param `overwrite=false`, in which case the request will fail if the configSet already exists.+====+--++Here is an example on how to upload a single file to a configset named "myConfigSet":++[.dynamic-tabs]+--+[example.tab-pane#v1uploadsinglefile]+====+[.tab-label]*V1 API*++With the v1 API, the `upload` command must be capitalized as `UPLOAD`.+The filename to upload is provided via the `filePath` URL param:++[source,bash]+----+curl -X POST --header "Content-Type:application/octet-stream"+ --data-binary @solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml+ "http://localhost:8983/solr/admin/configs?action=UPLOAD&name=myConfigSet&filePath=solrconfig.xml&overwrite=true"+----+====-NOTE: The `UPLOAD` command does not yet have a v2 equivalent API.+[example.tab-pane#v2uploadsinglefile]+====+[.tab-label]*V2 API*++With the v2 API, the name of the configset and file are both provided in the URL.+They can be substituted in `/cluster/configs/{config_name}/{file_name}`.+The filename may be nested and include `/` characters.++[source,bash]+----+curl -X PUT --header "Content-Type:application/octet-stream"+ --data-binary @solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml+ "http://localhost:8983/api/cluster/configs/myConfigSet/solrconfig.xml"+----++With this REST API, the default behavior is to overwrite the file if it already exists within the configSet.+This behavior can be disabled by providing the URL param `overwrite=false`, in which case the request will fail if the file already exists within the configSet.+====+--[[configsets-create]]== Create a Configset