Apache Solr open-source search software
Adding support for compression within the ZkCLI tool (#1348)
Justin Sweeney0e7a05de5656
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.javaindex f8ae0e194de..9cac8e45661 100644--- a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java+++ b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java@@ -25,7 +25,9 @@ import java.io.FileInputStream;import java.io.IOException;import java.io.InputStream;import java.io.PrintStream;+import java.lang.reflect.InvocationTargetException;import java.nio.charset.StandardCharsets;+import java.nio.file.Path;import java.nio.file.Paths;import java.util.List;import java.util.Properties;@@ -42,11 +44,17 @@ import org.apache.commons.cli.ParseException;import org.apache.commons.cli.PosixParser;import org.apache.commons.io.FileUtils;import org.apache.commons.io.IOUtils;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.StringUtils;import org.apache.solr.common.cloud.ClusterProperties;import org.apache.solr.common.cloud.SolrZkClient;import org.apache.solr.common.cloud.ZkMaintenanceUtils;+import org.apache.solr.common.util.Compressor;+import org.apache.solr.common.util.ZLibCompressor;import org.apache.solr.core.ConfigSetService;import org.apache.solr.core.CoreContainer;+import org.apache.solr.core.NodeConfig;+import org.apache.solr.core.SolrXmlConfig;import org.apache.solr.util.CLIO;import org.apache.zookeeper.CreateMode;import org.apache.zookeeper.KeeperException;@@ -234,6 +242,9 @@ public class ZkCLI implements CLIO {// start up a tmp zk server firstString zkServerAddress = line.getOptionValue(ZKHOST);String solrHome = line.getOptionValue(SOLRHOME);+ if (StringUtils.isEmpty(solrHome)) {+ solrHome = System.getProperty("solr.home");+ }String solrPort = null;if (line.hasOption(RUNZK)) {@@ -256,6 +267,38 @@ public class ZkCLI implements CLIO {zkServer.parseConfig();zkServer.start();}++ int minStateByteLenForCompression = -1;+ Compressor compressor = new ZLibCompressor();++ if (solrHome != null) {+ try {+ Path solrHomePath = Paths.get(solrHome);+ Properties props = new Properties();+ props.put(SolrXmlConfig.ZK_HOST, zkServerAddress);+ NodeConfig nodeConfig = NodeConfig.loadNodeConfig(solrHomePath, props);+ minStateByteLenForCompression =+ nodeConfig.getCloudConfig().getMinStateByteLenForCompression();+ String stateCompressorClass = nodeConfig.getCloudConfig().getStateCompressorClass();+ if (!StringUtils.isEmpty(stateCompressorClass)) {+ Class<? extends Compressor> compressionClass =+ Class.forName(stateCompressorClass).asSubclass(Compressor.class);+ compressor = compressionClass.getDeclaredConstructor().newInstance();+ }+ } catch (SolrException e) {+ // Failed to load solr.xml+ stdout.println(+ "Failed to load solr.xml from ZK or SolrHome, put/get operations on compressed data will use data as is. If you intention is to read and de-compress data or compress and write data, then solr.xml must be accessible.");+ } catch (ClassNotFoundException+ | NoSuchMethodException+ | InstantiationException+ | IllegalAccessException+ | InvocationTargetException e) {+ stdout.println("Unable to find or instantiate compression class: " + e.getMessage());+ System.exit(1);+ }+ }+SolrZkClient zkClient = null;try {zkClient =@@ -264,6 +307,7 @@ public class ZkCLI implements CLIO {.withTimeout(30000, TimeUnit.MILLISECONDS).withConnTimeOut(30000, TimeUnit.MILLISECONDS).withReconnectListener(() -> {})+ .withCompressor(compressor).build();if (line.getOptionValue(CMD).equalsIgnoreCase(BOOTSTRAP)) {@@ -359,16 +403,16 @@ public class ZkCLI implements CLIO {"-" + PUT + " requires two args - the path to create and the data string");System.exit(1);}- String path = arglist.get(0).toString();+ String path = arglist.get(0);+ byte[] data = arglist.get(1).getBytes(StandardCharsets.UTF_8);+ if (shouldCompressData(data, path, minStateByteLenForCompression)) {+ // state.json should be compressed before being put to ZK+ data = compressor.compressBytes(data);+ }if (zkClient.exists(path, true)) {- zkClient.setData(- path, arglist.get(1).toString().getBytes(StandardCharsets.UTF_8), true);+ zkClient.setData(path, data, true);} else {- zkClient.create(- path,- arglist.get(1).toString().getBytes(StandardCharsets.UTF_8),- CreateMode.PERSISTENT,- true);+ zkClient.create(path, data, CreateMode.PERSISTENT, true);}} else if (line.getOptionValue(CMD).equalsIgnoreCase(PUT_FILE)) {List<String> arglist = line.getArgList();@@ -382,11 +426,16 @@ public class ZkCLI implements CLIO {String path = arglist.get(0).toString();InputStream is = new FileInputStream(arglist.get(1).toString());+ byte[] data = IOUtils.toByteArray(is);+ if (shouldCompressData(data, path, minStateByteLenForCompression)) {+ // state.json should be compressed before being put to ZK+ data = compressor.compressBytes(data);+ }try {if (zkClient.exists(path, true)) {- zkClient.setData(path, IOUtils.toByteArray(is), true);+ zkClient.setData(path, data, true);} else {- zkClient.create(path, IOUtils.toByteArray(is), CreateMode.PERSISTENT, true);+ zkClient.create(path, data, CreateMode.PERSISTENT, true);}} finally {IOUtils.closeQuietly(is);@@ -450,4 +499,15 @@ public class ZkCLI implements CLIO {stdout.println("Unexpected exception:" + exp.getMessage());}}++ private static boolean shouldCompressData(+ byte[] data, String path, int minStateByteLenForCompression) {+ if (path.endsWith("state.json")+ && minStateByteLenForCompression > -1+ && data.length > minStateByteLenForCompression) {+ // state.json should be compressed before being put to ZK+ return true;+ }+ return false;+ }}diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java b/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.javaindex 58101f0c7c6..1ba4fde8caa 100644--- a/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java+++ b/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java@@ -32,6 +32,7 @@ import org.apache.commons.io.FileUtils;import org.apache.commons.io.IOUtils;import org.apache.commons.io.filefilter.RegexFileFilter;import org.apache.commons.io.filefilter.TrueFileFilter;+import org.apache.commons.lang3.StringUtils;import org.apache.solr.SolrJettyTestBase;import org.apache.solr.SolrTestCaseJ4;import org.apache.solr.common.SolrException;@@ -40,6 +41,7 @@ import org.apache.solr.common.cloud.SolrZkClient;import org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider;import org.apache.solr.common.cloud.ZkNodeProps;import org.apache.solr.common.cloud.ZkStateReader;+import org.apache.solr.common.util.ZLibCompressor;import org.apache.solr.core.ConfigSetService;import org.apache.solr.util.ExternalPaths;import org.apache.zookeeper.CreateMode;@@ -63,6 +65,8 @@ public class ZkCLITest extends SolrTestCaseJ4 {private SolrZkClient zkClient;+ private PrintStream originalSystemOut;+protected static final String SOLR_HOME = SolrTestCaseJ4.TEST_HOME();@BeforeClass@@ -87,6 +91,8 @@ public class ZkCLITest extends SolrTestCaseJ4 {Path tmpDir = createTempDir();solrHome = exampleHome;+ originalSystemOut = System.out;+zkDir = tmpDir.resolve("zookeeper/server1/data");log.info("ZooKeeper dataDir:{}", zkDir);zkServer = new ZkTestServer(zkDir);@@ -170,6 +176,28 @@ public class ZkCLITest extends SolrTestCaseJ4 {zkClient.getData("/data.txt", null, null, true), data.getBytes(StandardCharsets.UTF_8));}+ @Test+ public void testPutCompressed() throws Exception {+ // test put compressed+ System.setProperty("solr.home", solrHome);+ System.setProperty("minStateByteLenForCompression", "0");++ String data = "my data";+ ZLibCompressor zLibCompressor = new ZLibCompressor();+ byte[] expected = zLibCompressor.compressBytes(data.getBytes(StandardCharsets.UTF_8));+ String[] args =+ new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd", "put", "/state.json", data};+ ZkCLI.main(args);+ assertArrayEquals(zkClient.getZooKeeper().getData("/state.json", null, null), expected);++ // test re-put to existing+ data = "my data deux";+ expected = zLibCompressor.compressBytes(data.getBytes(StandardCharsets.UTF_8));+ args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd", "put", "/state.json", data};+ ZkCLI.main(args);+ assertArrayEquals(zkClient.getZooKeeper().getData("/state.json", null, null), expected);+ }+@Testpublic void testPutFile() throws Exception {// test put file@@ -197,6 +225,37 @@ public class ZkCLITest extends SolrTestCaseJ4 {assertEquals("Should get back what we put in ZK", fromZk, fromLoc);}+ @Test+ public void testPutFileCompressed() throws Exception {+ // test put file compressed+ System.setProperty("solr.home", solrHome);+ System.setProperty("minStateByteLenForCompression", "0");++ String[] args =+ new String[] {+ "-zkhost",+ zkServer.getZkAddress(),+ "-cmd",+ "putfile",+ "/state.json",+ SOLR_HOME + File.separator + "solr-stress-new.xml"+ };+ ZkCLI.main(args);++ byte[] fromZk = zkClient.getZooKeeper().getData("/state.json", null, null);+ File locFile = new File(SOLR_HOME + File.separator + "solr-stress-new.xml");+ InputStream is = new FileInputStream(locFile);+ byte[] fromLoc;+ try {+ fromLoc = IOUtils.toByteArray(is);+ ZLibCompressor zLibCompressor = new ZLibCompressor();+ fromLoc = zLibCompressor.compressBytes(fromLoc);+ } finally {+ IOUtils.closeQuietly(is);+ }+ assertArrayEquals("Should get back what we put in ZK", fromLoc, fromZk);+ }+@Testpublic void testPutFileNotExists() {// test put file@@ -379,9 +438,34 @@ public class ZkCLITest extends SolrTestCaseJ4 {public void testGet() throws Exception {String getNode = "/getNode";byte[] data = "getNode-data".getBytes(StandardCharsets.UTF_8);+ ByteArrayOutputStream systemOut = new ByteArrayOutputStream();this.zkClient.create(getNode, data, CreateMode.PERSISTENT, true);String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd", "get", getNode};+ ZkCLI.setStdout(new PrintStream(systemOut, true, StandardCharsets.UTF_8));ZkCLI.main(args);+ assertArrayEquals(+ data,+ StringUtils.removeEnd(systemOut.toString(StandardCharsets.UTF_8), "\n")+ .getBytes(StandardCharsets.UTF_8));+ }++ @Test+ public void testGetCompressed() throws Exception {+ System.setProperty("solr.home", solrHome);+ System.setProperty("minStateByteLenForCompression", "0");++ String getNode = "/getNode";+ byte[] data = "getNode-data".getBytes(StandardCharsets.UTF_8);+ ZLibCompressor zLibCompressor = new ZLibCompressor();+ ByteArrayOutputStream systemOut = new ByteArrayOutputStream();+ this.zkClient.create(getNode, zLibCompressor.compressBytes(data), CreateMode.PERSISTENT, true);+ String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd", "get", getNode};+ ZkCLI.setStdout(new PrintStream(systemOut, true, StandardCharsets.UTF_8));+ ZkCLI.main(args);+ assertArrayEquals(+ data,+ StringUtils.removeEnd(systemOut.toString(StandardCharsets.UTF_8), "\n")+ .getBytes(StandardCharsets.UTF_8));}@Test@@ -404,6 +488,27 @@ public class ZkCLITest extends SolrTestCaseJ4 {assertArrayEquals(data, readData);}+ @Test+ public void testGetFileCompressed() throws Exception {+ File tmpDir = createTempDir().toFile();++ String getNode = "/getFileNode";+ byte[] data = "getFileNode-data".getBytes(StandardCharsets.UTF_8);+ ZLibCompressor zLibCompressor = new ZLibCompressor();+ this.zkClient.create(getNode, zLibCompressor.compressBytes(data), CreateMode.PERSISTENT, true);++ File file =+ new File(tmpDir, "solrtest-getfile-" + this.getClass().getName() + "-" + System.nanoTime());+ String[] args =+ new String[] {+ "-zkhost", zkServer.getZkAddress(), "-cmd", "getfile", getNode, file.getAbsolutePath()+ };+ ZkCLI.main(args);++ byte[] readData = FileUtils.readFileToByteArray(file);+ assertArrayEquals(data, readData);+ }+@Testpublic void testGetFileNotExists() throws Exception {String getNode = "/getFileNotExistsNode";@@ -502,6 +607,9 @@ public class ZkCLITest extends SolrTestCaseJ4 {if (zkServer != null) {zkServer.shutdown();}+ System.clearProperty("solr.home");+ System.clearProperty("minStateByteLenForCompression");+ System.setOut(originalSystemOut);super.tearDown();}}diff --git a/solr/server/scripts/cloud-scripts/zkcli.sh b/solr/server/scripts/cloud-scripts/zkcli.shindex fa4e4556bfa..4477df48634 100755--- a/solr/server/scripts/cloud-scripts/zkcli.sh+++ b/solr/server/scripts/cloud-scripts/zkcli.sh@@ -11,6 +11,8 @@ sdir="`dirname \"$0\"`"log4j_config="file:$sdir/../../resources/log4j2-console.xml"+solr_home="$sdir/../../solr"+# Settings for ZK ACL#SOLR_ZK_CREDS_AND_ACLS="-DzkACLProvider=org.apache.solr.common.cloud.DigestZkACLProvider \# -DzkCredentialsProvider=org.apache.solr.common.cloud.DigestZkCredentialsProvider \@@ -21,6 +23,6 @@ log4j_config="file:$sdir/../../resources/log4j2-console.xml"#...# -DzkDigestCredentialsFile=/path/to/zkDigestCredentialsFile.properties#...-PATH=$JAVA_HOME/bin:$PATH $JVM $SOLR_ZK_CREDS_AND_ACLS $ZKCLI_JVM_FLAGS -Dlog4j.configurationFile=$log4j_config \+PATH=$JAVA_HOME/bin:$PATH $JVM $SOLR_ZK_CREDS_AND_ACLS $ZKCLI_JVM_FLAGS -Dlog4j.configurationFile=$log4j_config -DsolrHome=$solr_home \-classpath "$sdir/../../solr-webapp/webapp/WEB-INF/lib/*:$sdir/../../lib/ext/*:$sdir/../../lib/*" org.apache.solr.cloud.ZkCLI ${1+"$@"}